diff --git a/cereal/custom.capnp b/cereal/custom.capnp index 06d3eec0b3..b2575256d8 100644 --- a/cereal/custom.capnp +++ b/cereal/custom.capnp @@ -27,7 +27,53 @@ struct SelfdriveStateSP @0x81c2f05a394cf4af { } } -struct CustomReserved1 @0xaedffd8f31e7b55d { +struct ModelManagerSP @0xaedffd8f31e7b55d { + activeBundle @0 :ModelBundle; + selectedBundle @1 :ModelBundle; + availableBundles @2 :List(ModelBundle); + + struct DownloadUri { + uri @0 :Text; + sha256 @1 :Text; + } + + enum Type { + drive @0; + navigation @1; + metadata @2; + } + + struct Model { + fullName @0 :Text; + fileName @1 :Text; + downloadUri @2 :DownloadUri; + downloadProgress @3 :DownloadProgress; + type @4 :Type; + } + + enum DownloadStatus { + notDownloading @0; + downloading @1; + downloaded @2; + cached @3; + failed @4; + } + + struct DownloadProgress { + status @0 :DownloadStatus; + progress @1 :Float32; + eta @2 :UInt32; + } + + struct ModelBundle { + index @0 :UInt32; + internalName @1 :Text; + displayName @2 :Text; + models @3 :List(Model); + status @4 :DownloadStatus; + generation @5 :UInt32; + environment @6 :Text; + } } struct CustomReserved2 @0xf35cc4560bbf6ec2 { diff --git a/cereal/log.capnp b/cereal/log.capnp index 50d4d8e4dd..d5fbad6fe8 100644 --- a/cereal/log.capnp +++ b/cereal/log.capnp @@ -2632,7 +2632,7 @@ struct Event { # *********** Custom: reserved for forks *********** selfdriveStateSP @107 :Custom.SelfdriveStateSP; - customReserved1 @108 :Custom.CustomReserved1; + modelManagerSP @108 :Custom.ModelManagerSP; customReserved2 @109 :Custom.CustomReserved2; customReserved3 @110 :Custom.CustomReserved3; customReserved4 @111 :Custom.CustomReserved4; diff --git a/cereal/services.py b/cereal/services.py index c146190ee7..346bf81a09 100755 --- a/cereal/services.py +++ b/cereal/services.py @@ -75,6 +75,7 @@ _services: dict[str, tuple] = { "microphone": (True, 10., 10), # sunnypilot + "modelManagerSP": (False, 1., 1), "selfdriveStateSP": (True, 100., 10), # debug diff --git a/common/params.cc b/common/params.cc index 1e845635d9..22b8c5f119 100644 --- a/common/params.cc +++ b/common/params.cc @@ -202,12 +202,15 @@ std::unordered_map keys = { {"Version", PERSISTENT}, // sunnypilot params - {"Mads", PERSISTENT}, + {"EnableGithubRunner", PERSISTENT}, + {"HyundaiLongitudinalMainCruiseToggleable", PERSISTENT}, {"MadsMainCruiseAllowed", PERSISTENT}, {"MadsPauseLateralOnBrake", PERSISTENT}, {"MadsUnifiedEngagementMode", PERSISTENT}, - {"HyundaiLongitudinalMainCruiseToggleable", PERSISTENT}, - {"EnableGithubRunner", PERSISTENT}, + {"ModelManager_ActiveBundle", PERSISTENT}, + {"ModelManager_DownloadIndex", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_ONROAD_TRANSITION}, + {"ModelManager_LastSyncTime", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION}, + {"ModelManager_ModelsCache", PERSISTENT}, }; } // namespace diff --git a/selfdrive/ui/qt/offroad/settings.h b/selfdrive/ui/qt/offroad/settings.h index 41914c0ec0..b5b148e9a4 100644 --- a/selfdrive/ui/qt/offroad/settings.h +++ b/selfdrive/ui/qt/offroad/settings.h @@ -94,7 +94,7 @@ public: protected: void showEvent(QShowEvent *event) override; - void updateLabels(); + virtual void updateLabels(); void checkForUpdates(); bool is_onroad = false; diff --git a/selfdrive/ui/sunnypilot/SConscript b/selfdrive/ui/sunnypilot/SConscript index d9f319baf8..3e3bda6b62 100644 --- a/selfdrive/ui/sunnypilot/SConscript +++ b/selfdrive/ui/sunnypilot/SConscript @@ -10,6 +10,7 @@ qt_src = [ "sunnypilot/qt/window.cc", "sunnypilot/qt/home.cc", "sunnypilot/qt/offroad/settings/settings.cc", + "sunnypilot/qt/offroad/settings/software_panel.cc", "sunnypilot/qt/offroad/settings/sunnypilot_panel.cc", "sunnypilot/qt/onroad/onroad_home.cc", ] diff --git a/selfdrive/ui/sunnypilot/qt/offroad/settings/settings.cc b/selfdrive/ui/sunnypilot/qt/offroad/settings/settings.cc index ab1b0e23b7..cd7657e540 100644 --- a/selfdrive/ui/sunnypilot/qt/offroad/settings/settings.cc +++ b/selfdrive/ui/sunnypilot/qt/offroad/settings/settings.cc @@ -11,6 +11,7 @@ #include "selfdrive/ui/sunnypilot/qt/widgets/scrollview.h" #include "selfdrive/ui/qt/offroad/developer_panel.h" +#include "selfdrive/ui/sunnypilot/qt/offroad/settings/software_panel.h" #include "selfdrive/ui/sunnypilot/qt/offroad/settings/sunnypilot_panel.h" TogglesPanelSP::TogglesPanelSP(SettingsWindow *parent) : TogglesPanel(parent) { @@ -69,7 +70,7 @@ SettingsWindowSP::SettingsWindowSP(QWidget *parent) : SettingsWindow(parent) { PanelInfo(" " + tr("Device"), device, "../../sunnypilot/selfdrive/assets/offroad/icon_home.svg"), PanelInfo(" " + tr("Network"), networking, "../assets/offroad/icon_network.png"), PanelInfo(" " + tr("Toggles"), toggles, "../../sunnypilot/selfdrive/assets/offroad/icon_toggle.png"), - PanelInfo(" " + tr("Software"), new SoftwarePanel(this), "../../sunnypilot/selfdrive/assets/offroad/icon_software.png"), + PanelInfo(" " + tr("Software"), new SoftwarePanelSP(this), "../../sunnypilot/selfdrive/assets/offroad/icon_software.png"), PanelInfo(" " + tr("sunnypilot"), new SunnypilotPanel(this), "../assets/images/button_home.png"), PanelInfo(" " + tr("Developer"), new DeveloperPanel(this), "../assets/offroad/icon_shell.png"), }; diff --git a/selfdrive/ui/sunnypilot/qt/offroad/settings/software_panel.cc b/selfdrive/ui/sunnypilot/qt/offroad/settings/software_panel.cc new file mode 100644 index 0000000000..6bbef9fb94 --- /dev/null +++ b/selfdrive/ui/sunnypilot/qt/offroad/settings/software_panel.cc @@ -0,0 +1,180 @@ +/** + * Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. + * + * This file is part of sunnypilot and is licensed under the MIT License. + * See the LICENSE.md file in the root directory for more details. + */ + +#include "selfdrive/ui/sunnypilot/qt/offroad/settings/software_panel.h" + +#include +#include + +/** + * @brief Constructs the software panel with model bundle selection functionality + * @param parent Parent widget + */ +SoftwarePanelSP::SoftwarePanelSP(QWidget *parent) : SoftwarePanel(parent) { + const auto current_model = GetActiveModelName(); + currentModelLblBtn = new ButtonControlSP(tr("Current Model"), tr("SELECT"), current_model); + currentModelLblBtn->setValue(current_model); + + connect(currentModelLblBtn, &ButtonControlSP::clicked, this, &SoftwarePanelSP::handleCurrentModelLblBtnClicked); + QObject::connect(uiStateSP(), &UIStateSP::uiUpdate, this, &SoftwarePanelSP::updateLabels); + AddWidgetAt(0, currentModelLblBtn); +} + +/** + * @brief Updates the UI with bundle download progress information + * Reads status from modelManagerSP cereal message and displays status for all models + */ +void SoftwarePanelSP::handleBundleDownloadProgress() { + const SubMaster &sm = *(uiStateSP()->sm); + const auto model_manager = sm["modelManagerSP"].getModelManagerSP(); + + if (!model_manager.hasSelectedBundle()) { + currentModelLblBtn->setDescription(""); + return; + } + + const auto &bundle = model_manager.getSelectedBundle(); + const auto &models = bundle.getModels(); + QStringList status; + + // Get status for each model type in order + for (const auto &model: models) { + QString typeName; + QString modelName; + + switch (model.getType()) { + case cereal::ModelManagerSP::Type::DRIVE: + typeName = tr("Driving"); + modelName = QString::fromStdString(bundle.getDisplayName()); + break; + case cereal::ModelManagerSP::Type::NAVIGATION: + typeName = tr("Navigation"); + modelName = QString::fromStdString(model.getFullName()); + break; + case cereal::ModelManagerSP::Type::METADATA: + typeName = tr("Metadata"); + modelName = QString::fromStdString(model.getFullName()); + break; + } + + const auto &progress = model.getDownloadProgress(); + QString line; + + if (progress.getStatus() == cereal::ModelManagerSP::DownloadStatus::DOWNLOADING) { + line = tr("Downloading %1 model [%2]... (%3%)").arg(typeName, modelName).arg(progress.getProgress(), 0, 'f', 2); + } else if (progress.getStatus() == cereal::ModelManagerSP::DownloadStatus::DOWNLOADED) { + line = tr("%1 model [%2] downloaded").arg(typeName, modelName); + } else if (progress.getStatus() == cereal::ModelManagerSP::DownloadStatus::CACHED) { + line = tr("%1 model [%2] from cache").arg(typeName, modelName); + } else if (progress.getStatus() == cereal::ModelManagerSP::DownloadStatus::FAILED) { + line = tr("%1 model [%2] download failed").arg(typeName, modelName); + } else { + line = tr("%1 model [%2] pending...").arg(typeName, modelName); + } + status.append(line); + } + + currentModelLblBtn->setDescription(status.join("\n")); + + if (bundle.getStatus() == cereal::ModelManagerSP::DownloadStatus::DOWNLOADING) { + currentModelLblBtn->showDescription(); + } + + currentModelLblBtn->setEnabled(!is_onroad && !isDownloading()); +} + +/** + * @brief Gets the name of the currently selected model bundle + * @return Display name of the selected bundle or default model name + */ +QString SoftwarePanelSP::GetActiveModelName() { + const SubMaster &sm = *(uiStateSP()->sm); + const auto model_manager = sm["modelManagerSP"].getModelManagerSP(); + + if (model_manager.hasActiveBundle()) { + return QString::fromStdString(model_manager.getActiveBundle().getDisplayName()); + } + + return ""; +} + +/** + * @brief Handles the model bundle selection button click + * Displays available bundles, allows selection, and initiates download + */ +void SoftwarePanelSP::handleCurrentModelLblBtnClicked() { + currentModelLblBtn->setEnabled(false); + currentModelLblBtn->setValue(tr("Fetching models...")); + + const SubMaster &sm = *(uiStateSP()->sm); + const auto model_manager = sm["modelManagerSP"].getModelManagerSP(); + + // Create mapping of bundle indices to display names + QMap index_to_bundle; + const auto bundles = model_manager.getAvailableBundles(); + for (const auto &bundle: bundles) { + index_to_bundle.insert(bundle.getIndex(), QString::fromStdString(bundle.getDisplayName())); + } + + // Sort bundles by index in descending order + QStringList bundleNames; + auto indices = index_to_bundle.keys(); + std::sort(indices.begin(), indices.end(), std::greater()); + for (const auto &index: indices) { + bundleNames.append(index_to_bundle[index]); + } + + currentModelLblBtn->setEnabled(!is_onroad); + currentModelLblBtn->setValue(GetActiveModelName()); + + const QString selectedBundleName = MultiOptionDialog::getSelection( + tr("Select a Model"), bundleNames, GetActiveModelName(), this); + + if (selectedBundleName.isEmpty() || !canContinueOnMeteredDialog()) { + return; + } + + // Find selected bundle and initiate download + for (const auto &bundle: bundles) { + if (QString::fromStdString(bundle.getDisplayName()) == selectedBundleName) { + params.put("ModelManager_DownloadIndex", std::to_string(bundle.getIndex())); + if (bundle.getGeneration() != model_manager.getActiveBundle().getGeneration()) { + showResetParamsDialog(); + } + break; + } + } + + updateLabels(); +} + +/** + * @brief Updates the UI elements based on current state + */ +void SoftwarePanelSP::updateLabels() { + if (!isVisible()) { + return; + } + + handleBundleDownloadProgress(); + currentModelLblBtn->setValue(GetActiveModelName()); + SoftwarePanel::updateLabels(); +} + +/** + * @brief Shows dialog prompting user to reset calibration after model download + */ +void SoftwarePanelSP::showResetParamsDialog() { + const auto confirmMsg = tr("Model download has started in the background.") + "\n" + + tr("We STRONGLY suggest you to reset calibration. Would you like to do that now?"); + const auto button_text = tr("Reset Calibration"); + + if (showConfirmationDialog(confirmMsg, button_text, false)) { + params.remove("CalibrationParams"); + params.remove("LiveTorqueParameters"); + } +} diff --git a/selfdrive/ui/sunnypilot/qt/offroad/settings/software_panel.h b/selfdrive/ui/sunnypilot/qt/offroad/settings/software_panel.h new file mode 100644 index 0000000000..fd837e31fa --- /dev/null +++ b/selfdrive/ui/sunnypilot/qt/offroad/settings/software_panel.h @@ -0,0 +1,61 @@ +/** + * Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. + * + * This file is part of sunnypilot and is licensed under the MIT License. + * See the LICENSE.md file in the root directory for more details. + */ + +#pragma once + +#include +#include "selfdrive/ui/sunnypilot/ui.h" +#include "selfdrive/ui/qt/offroad/settings.h" + +class SoftwarePanelSP final : public SoftwarePanel { + Q_OBJECT + +public: + explicit SoftwarePanelSP(QWidget *parent = nullptr); + +private: + QString GetActiveModelName(); + + bool isDownloading() const { + const SubMaster &sm = *(uiStateSP()->sm); + const auto model_manager = sm["modelManagerSP"].getModelManagerSP(); + + if (!model_manager.hasSelectedBundle()) { + return false; + } + + const auto &selected_bundle = model_manager.getSelectedBundle(); + return selected_bundle.getStatus() == cereal::ModelManagerSP::DownloadStatus::DOWNLOADING; + } + + // UI update related methods + void updateLabels() override; + void handleCurrentModelLblBtnClicked(); + void handleBundleDownloadProgress(); + void showResetParamsDialog(); + + bool canContinueOnMeteredDialog() { + if (!is_metered) return true; + return showConfirmationDialog(QString(), QString(), is_metered); + } + + inline bool showConfirmationDialog(const QString &message = QString(), const QString &confirmButtonText = QString(), const bool show_metered_warning = false) { + return showConfirmationDialog(this, message, confirmButtonText, show_metered_warning); + } + + static inline bool showConfirmationDialog(QWidget *parent, const QString &message = QString(), const QString &confirmButtonText = QString(), const bool show_metered_warning = false) { + const QString warning_message = show_metered_warning ? tr("Warning: You are on a metered connection!") : QString(); + const QString final_message = QString("%1%2").arg(!message.isEmpty() ? message + "\n" : QString(), warning_message); + const QString final_buttonText = !confirmButtonText.isEmpty() ? confirmButtonText : QString(tr("Continue") + " %1").arg(show_metered_warning ? tr("on Metered") : ""); + + return ConfirmationDialog::confirm(final_message, final_buttonText, parent); + } + + bool is_metered{}; + bool is_wifi{}; + ButtonControlSP *currentModelLblBtn; +}; diff --git a/selfdrive/ui/sunnypilot/ui.cc b/selfdrive/ui/sunnypilot/ui.cc index 424c395600..b27c839805 100644 --- a/selfdrive/ui/sunnypilot/ui.cc +++ b/selfdrive/ui/sunnypilot/ui.cc @@ -18,7 +18,7 @@ UIStateSP::UIStateSP(QObject *parent) : UIState(parent) { "modelV2", "controlsState", "liveCalibration", "radarState", "deviceState", "pandaStates", "carParams", "driverMonitoringState", "carState", "driverStateV2", "wideRoadCameraState", "managerState", "selfdriveState", "longitudinalPlan", - "selfdriveStateSP", + "modelManagerSP", "selfdriveStateSP", }); // update timer diff --git a/selfdrive/ui/translations/main_ar.ts b/selfdrive/ui/translations/main_ar.ts index 8d1be1437d..11bcb2155b 100644 --- a/selfdrive/ui/translations/main_ar.ts +++ b/selfdrive/ui/translations/main_ar.ts @@ -986,6 +986,81 @@ This may take up to a minute. أحدث نسخة، آخر تحقق %1 + + SoftwarePanelSP + + SELECT + اختيار + + + We STRONGLY suggest you to reset calibration. Would you like to do that now? + + + + Reset Calibration + إعادة ضبط المعايرة + + + Warning: You are on a metered connection! + + + + Continue + متابعة + + + on Metered + + + + %1 model [%2] pending... + + + + Current Model + + + + Driving + + + + Navigation + + + + Metadata + + + + Downloading %1 model [%2]... (%3%) + + + + %1 model [%2] downloaded + + + + %1 model [%2] download failed + + + + %1 model [%2] from cache + + + + Select a Model + + + + Fetching models... + + + + Model download has started in the background. + + + SshControl diff --git a/selfdrive/ui/translations/main_de.ts b/selfdrive/ui/translations/main_de.ts index cfed9a3452..447246aad8 100644 --- a/selfdrive/ui/translations/main_de.ts +++ b/selfdrive/ui/translations/main_de.ts @@ -970,6 +970,81 @@ This may take up to a minute. + + SoftwarePanelSP + + SELECT + AUSWÄHLEN + + + We STRONGLY suggest you to reset calibration. Would you like to do that now? + + + + Reset Calibration + Neu kalibrieren + + + Warning: You are on a metered connection! + + + + Continue + Fortsetzen + + + on Metered + + + + %1 model [%2] pending... + + + + Current Model + + + + Driving + + + + Navigation + + + + Metadata + + + + Downloading %1 model [%2]... (%3%) + + + + %1 model [%2] downloaded + + + + %1 model [%2] download failed + + + + %1 model [%2] from cache + + + + Select a Model + + + + Fetching models... + + + + Model download has started in the background. + + + SshControl diff --git a/selfdrive/ui/translations/main_es.ts b/selfdrive/ui/translations/main_es.ts index 7e4c4d7d86..2c7eac5702 100644 --- a/selfdrive/ui/translations/main_es.ts +++ b/selfdrive/ui/translations/main_es.ts @@ -970,6 +970,81 @@ Esto puede tardar un minuto. actualizado, último chequeo %1 + + SoftwarePanelSP + + SELECT + SELECCIONAR + + + We STRONGLY suggest you to reset calibration. Would you like to do that now? + + + + Reset Calibration + Formatear Calibración + + + Warning: You are on a metered connection! + + + + Continue + Continuar + + + on Metered + + + + %1 model [%2] pending... + modelo de %1 [%2] pendiente... + + + Fetching models... + Obteniendo modelos + + + Model download has started in the background. + Descarga de modelo iniciada en segundo plano. + + + Current Model + Modelo Actual + + + Driving + Conducción + + + Navigation + Navegación + + + Metadata + Metadatos + + + Downloading %1 model [%2]... (%3%) + Descargando modelo de %1 [%2]... (%3%) + + + %1 model [%2] downloaded + Modelo de %1 [%2] descargado + + + %1 model [%2] download failed + Falló descarga modelo de %1 [%2] + + + %1 model [%2] from cache + Modelo de %1 [%2] desde caché + + + Select a Model + Selecciona un Modelo + + SshControl diff --git a/selfdrive/ui/translations/main_fr.ts b/selfdrive/ui/translations/main_fr.ts index 3851e61f77..351e684a21 100644 --- a/selfdrive/ui/translations/main_fr.ts +++ b/selfdrive/ui/translations/main_fr.ts @@ -970,6 +970,81 @@ Cela peut prendre jusqu'à une minute. à jour, dernière vérification %1 + + SoftwarePanelSP + + SELECT + SÉLECTIONNER + + + We STRONGLY suggest you to reset calibration. Would you like to do that now? + + + + Reset Calibration + Réinitialiser la calibration + + + Warning: You are on a metered connection! + + + + Continue + Continuer + + + on Metered + + + + %1 model [%2] pending... + + + + Current Model + + + + Driving + + + + Navigation + + + + Metadata + + + + Downloading %1 model [%2]... (%3%) + + + + %1 model [%2] downloaded + + + + %1 model [%2] download failed + + + + %1 model [%2] from cache + + + + Select a Model + + + + Fetching models... + + + + Model download has started in the background. + + + SshControl diff --git a/selfdrive/ui/translations/main_ja.ts b/selfdrive/ui/translations/main_ja.ts index dc8e2f529e..f1690bb76d 100644 --- a/selfdrive/ui/translations/main_ja.ts +++ b/selfdrive/ui/translations/main_ja.ts @@ -964,6 +964,81 @@ This may take up to a minute. + + SoftwarePanelSP + + SELECT + 選択 + + + We STRONGLY suggest you to reset calibration. Would you like to do that now? + + + + Reset Calibration + ｷｬﾘﾌﾞﾚｰｼｮﾝをﾘｾｯﾄ + + + Warning: You are on a metered connection! + + + + Continue + 続ける + + + on Metered + + + + %1 model [%2] pending... + + + + Current Model + + + + Driving + + + + Navigation + + + + Metadata + + + + Downloading %1 model [%2]... (%3%) + + + + %1 model [%2] downloaded + + + + %1 model [%2] download failed + + + + %1 model [%2] from cache + + + + Select a Model + + + + Fetching models... + + + + Model download has started in the background. + + + SshControl diff --git a/selfdrive/ui/translations/main_ko.ts b/selfdrive/ui/translations/main_ko.ts index d6f1de1912..2dc036fee9 100644 --- a/selfdrive/ui/translations/main_ko.ts +++ b/selfdrive/ui/translations/main_ko.ts @@ -966,6 +966,81 @@ This may take up to a minute. 업데이트 안함 + + SoftwarePanelSP + + SELECT + 선택 + + + We STRONGLY suggest you to reset calibration. Would you like to do that now? + + + + Reset Calibration + 캘리브레이션 초기화 + + + Warning: You are on a metered connection! + + + + Continue + 계속 + + + on Metered + + + + %1 model [%2] pending... + + + + Current Model + + + + Driving + + + + Navigation + + + + Metadata + + + + Downloading %1 model [%2]... (%3%) + + + + %1 model [%2] downloaded + + + + %1 model [%2] download failed + + + + %1 model [%2] from cache + + + + Select a Model + + + + Fetching models... + + + + Model download has started in the background. + + + SshControl diff --git a/selfdrive/ui/translations/main_pt-BR.ts b/selfdrive/ui/translations/main_pt-BR.ts index bd87b1a7f1..ff8a2efeff 100644 --- a/selfdrive/ui/translations/main_pt-BR.ts +++ b/selfdrive/ui/translations/main_pt-BR.ts @@ -970,6 +970,81 @@ Isso pode levar até um minuto. nunca + + SoftwarePanelSP + + SELECT + SELECIONE + + + We STRONGLY suggest you to reset calibration. Would you like to do that now? + + + + Reset Calibration + Reinicializar Calibragem + + + Warning: You are on a metered connection! + + + + Continue + Continuar + + + on Metered + + + + %1 model [%2] pending... + + + + Current Model + + + + Driving + + + + Navigation + + + + Metadata + + + + Downloading %1 model [%2]... (%3%) + + + + %1 model [%2] downloaded + + + + %1 model [%2] download failed + + + + %1 model [%2] from cache + + + + Select a Model + + + + Fetching models... + + + + Model download has started in the background. + + + SshControl diff --git a/selfdrive/ui/translations/main_th.ts b/selfdrive/ui/translations/main_th.ts index 4d08e85e47..1162f64404 100644 --- a/selfdrive/ui/translations/main_th.ts +++ b/selfdrive/ui/translations/main_th.ts @@ -966,6 +966,81 @@ This may take up to a minute. ล่าสุดแล้ว ตรวจสอบครั้งสุดท้ายเมื่อ %1 + + SoftwarePanelSP + + SELECT + เลือก + + + We STRONGLY suggest you to reset calibration. Would you like to do that now? + + + + Reset Calibration + รีเซ็ตการคาลิเบรท + + + Warning: You are on a metered connection! + + + + Continue + ดำเนินการต่อ + + + on Metered + + + + %1 model [%2] pending... + + + + Current Model + + + + Driving + + + + Navigation + + + + Metadata + + + + Downloading %1 model [%2]... (%3%) + + + + %1 model [%2] downloaded + + + + %1 model [%2] download failed + + + + %1 model [%2] from cache + + + + Select a Model + + + + Fetching models... + + + + Model download has started in the background. + + + SshControl diff --git a/selfdrive/ui/translations/main_tr.ts b/selfdrive/ui/translations/main_tr.ts index caf4b97766..fbc9292bc6 100644 --- a/selfdrive/ui/translations/main_tr.ts +++ b/selfdrive/ui/translations/main_tr.ts @@ -964,6 +964,81 @@ This may take up to a minute. + + SoftwarePanelSP + + SELECT + + + + We STRONGLY suggest you to reset calibration. Would you like to do that now? + + + + Reset Calibration + Kalibrasyonu sıfırla + + + Warning: You are on a metered connection! + + + + Continue + Devam et + + + on Metered + + + + %1 model [%2] pending... + + + + Current Model + + + + Driving + + + + Navigation + + + + Metadata + + + + Downloading %1 model [%2]... (%3%) + + + + %1 model [%2] downloaded + + + + %1 model [%2] download failed + + + + %1 model [%2] from cache + + + + Select a Model + + + + Fetching models... + + + + Model download has started in the background. + + + SshControl diff --git a/selfdrive/ui/translations/main_zh-CHS.ts b/selfdrive/ui/translations/main_zh-CHS.ts index 0b2c45c1c1..7b5a700a55 100644 --- a/selfdrive/ui/translations/main_zh-CHS.ts +++ b/selfdrive/ui/translations/main_zh-CHS.ts @@ -966,6 +966,81 @@ This may take up to a minute. 从未更新 + + SoftwarePanelSP + + SELECT + 选择 + + + We STRONGLY suggest you to reset calibration. Would you like to do that now? + + + + Reset Calibration + 重置设备校准 + + + Warning: You are on a metered connection! + + + + Continue + 继续 + + + on Metered + + + + %1 model [%2] pending... + + + + Current Model + + + + Driving + + + + Navigation + + + + Metadata + + + + Downloading %1 model [%2]... (%3%) + + + + %1 model [%2] downloaded + + + + %1 model [%2] download failed + + + + %1 model [%2] from cache + + + + Select a Model + + + + Fetching models... + + + + Model download has started in the background. + + + SshControl diff --git a/selfdrive/ui/translations/main_zh-CHT.ts b/selfdrive/ui/translations/main_zh-CHT.ts index c54f724756..b3f04483ce 100644 --- a/selfdrive/ui/translations/main_zh-CHT.ts +++ b/selfdrive/ui/translations/main_zh-CHT.ts @@ -966,6 +966,81 @@ This may take up to a minute. 從未更新 + + SoftwarePanelSP + + SELECT + 選取 + + + We STRONGLY suggest you to reset calibration. Would you like to do that now? + + + + Reset Calibration + 重設校準 + + + Warning: You are on a metered connection! + + + + Continue + 繼續 + + + on Metered + + + + %1 model [%2] pending... + + + + Current Model + + + + Driving + + + + Navigation + + + + Metadata + + + + Downloading %1 model [%2]... (%3%) + + + + %1 model [%2] downloaded + + + + %1 model [%2] download failed + + + + %1 model [%2] from cache + + + + Select a Model + + + + Fetching models... + + + + Model download has started in the background. + + + SshControl diff --git a/sunnypilot/__init__.py b/sunnypilot/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sunnypilot/models/__init__.py b/sunnypilot/models/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sunnypilot/models/fetcher.py b/sunnypilot/models/fetcher.py new file mode 100644 index 0000000000..67f7efbae9 --- /dev/null +++ b/sunnypilot/models/fetcher.py @@ -0,0 +1,158 @@ +# Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. +# +# This file is part of sunnypilot and is licensed under the MIT License. +# See the LICENSE.md file in the root directory for more details. + +import json +import time + +import requests +from openpilot.common.params import Params +from openpilot.common.swaglog import cloudlog + +from cereal import custom + + +class ModelParser: + """Handles parsing of model data into cereal objects""" + + @staticmethod + def _parse_model(full_name: str, file_name: str, uri_data: dict, + model_type: custom.ModelManagerSP.Type) -> custom.ModelManagerSP.Model: + model = custom.ModelManagerSP.Model() + download_uri = custom.ModelManagerSP.DownloadUri() + + download_uri.uri = uri_data["url"] + download_uri.sha256 = uri_data["sha256"] + + model.fullName = full_name + model.fileName = file_name + model.downloadUri = download_uri + model.type = model_type + + return model + + @staticmethod + def _parse_bundle(key: str, value: dict) -> custom.ModelManagerSP.ModelBundle: + model_bundle = custom.ModelManagerSP.ModelBundle() + + # Parse main driving model + models = [ + ModelParser._parse_model( + value["full_name"], + value["file_name"], + value["download_uri"], + custom.ModelManagerSP.Type.drive + ) + ] + + # Parse navigation model if exists + if value.get("download_uri_nav"): + models.append(ModelParser._parse_model( + value["full_name_nav"], + value["file_name_nav"], + value["download_uri_nav"], + custom.ModelManagerSP.Type.navigation + )) + + # Parse metadata model if exists + if value.get("download_uri_metadata"): + models.append(ModelParser._parse_model( + value["full_name_metadata"], + value["file_name_metadata"], + value["download_uri_metadata"], + custom.ModelManagerSP.Type.metadata + )) + + model_bundle.index = int(value["index"]) + model_bundle.internalName = key + model_bundle.displayName = value["display_name"] + model_bundle.models = models + model_bundle.status = 0 + model_bundle.generation = int(value["generation"]) + model_bundle.environment = value["environment"] + + return model_bundle + + @staticmethod + def parse_models(json_data: dict) -> list[custom.ModelManagerSP.ModelBundle]: + return [ModelParser._parse_bundle(key, value) for key, value in json_data.items()] + + +class ModelCache: + """Handles caching of model data to avoid frequent remote fetches""" + + def __init__(self, params: Params, cache_timeout: int = int(3600 * 1e9)): + self.params = params + self.cache_timeout = cache_timeout + self._LAST_SYNC_KEY = "ModelManager_LastSyncTime" + self._CACHE_KEY = "ModelManager_ModelsCache" + + def _is_expired(self) -> bool: + """Checks if the cache has expired""" + current_time = int(time.monotonic() * 1e9) + last_sync = int(self.params.get(self._LAST_SYNC_KEY, encoding="utf-8") or 0) + return (current_time - last_sync) >= self.cache_timeout + + def get(self) -> tuple[dict, bool]: + """ + Retrieves cached model data and expiration status atomically. + Returns: Tuple of (cached_data, is_expired) + If no cached data exists or on error, returns an empty dict + """ + try: + cached_data = self.params.get(self._CACHE_KEY, encoding="utf-8") + if not cached_data: + cloudlog.warning("No cached model data available") + return {}, True + return json.loads(cached_data), self._is_expired() + except Exception as e: + cloudlog.exception(f"Error retrieving cached model data: {str(e)}") + return {}, True + + def set(self, data: dict) -> None: + """Updates the cache with new model data""" + self.params.put(self._CACHE_KEY, json.dumps(data)) + self.params.put(self._LAST_SYNC_KEY, str(int(time.monotonic() * 1e9))) + + +class ModelFetcher: + """Handles fetching and caching of model data from remote source""" + MODEL_URL = "https://docs.sunnypilot.ai/driving_models.json" + + def __init__(self, params: Params): + self.params = params + self.model_cache = ModelCache(params) + self.model_parser = ModelParser() + + def _fetch_and_cache_models(self) -> list[custom.ModelManagerSP.ModelBundle]: + """Fetches fresh model data from remote and updates cache""" + try: + response = requests.get(self.MODEL_URL, timeout=10) + response.raise_for_status() + json_data = response.json() + + self.model_cache.set(json_data) + cloudlog.debug("Successfully updated models cache") + return self.model_parser.parse_models(json_data) + except Exception: + cloudlog.exception("Error fetching models") + raise + + def get_available_models(self) -> list[custom.ModelManagerSP.ModelBundle]: + """Gets the list of available models, with smart cache handling""" + cached_data, is_expired = self.model_cache.get() + + if cached_data and not is_expired: + cloudlog.debug("Using valid cached models data") + return self.model_parser.parse_models(cached_data) + + try: + return self._fetch_and_cache_models() + except Exception: + if not cached_data: + cloudlog.exception("Failed to fetch fresh data and no cache available") + raise + + cloudlog.warning("Failed to fetch fresh data. Using expired cache as fallback") + return self.model_parser.parse_models(cached_data) diff --git a/sunnypilot/models/helpers.py b/sunnypilot/models/helpers.py new file mode 100644 index 0000000000..fe6ac2133c --- /dev/null +++ b/sunnypilot/models/helpers.py @@ -0,0 +1,32 @@ +# Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. +# +# This file is part of sunnypilot and is licensed under the MIT License. +# See the LICENSE.md file in the root directory for more details. + +import hashlib +import os +from openpilot.common.params import Params +from cereal import custom, messaging + + +async def verify_file(file_path: str, expected_hash: str) -> bool: + """Verifies file hash against expected hash""" + if not os.path.exists(file_path): + return False + + sha256_hash = hashlib.sha256() + with open(file_path, "rb") as file: + for chunk in iter(lambda: file.read(4096), b""): + sha256_hash.update(chunk) + + return sha256_hash.hexdigest().lower() == expected_hash.lower() + +def get_active_bundle(params: Params) -> custom.ModelManagerSP.ModelBundle: + """Gets the active model bundle from cache""" + if params is None: + params = Params() + + if active_bundle := params.get("ModelManager_ActiveBundle"): + return messaging.log_from_bytes(active_bundle, custom.ModelManagerSP.ModelBundle) + + return None diff --git a/sunnypilot/models/manager.py b/sunnypilot/models/manager.py new file mode 100644 index 0000000000..4226c6c679 --- /dev/null +++ b/sunnypilot/models/manager.py @@ -0,0 +1,179 @@ +# Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. +# +# This file is part of sunnypilot and is licensed under the MIT License. +# See the LICENSE.md file in the root directory for more details. + +import asyncio +import os +import time + +import aiohttp +from openpilot.common.params import Params +from openpilot.common.realtime import Ratekeeper +from openpilot.common.swaglog import cloudlog +from openpilot.system.hardware.hw import Paths + +from cereal import messaging, custom +from sunnypilot.models.fetcher import ModelFetcher +from sunnypilot.models.helpers import verify_file, get_active_bundle + + +class ModelManagerSP: + """Manages model downloads and status reporting""" + + def __init__(self): + self.params = Params() + self.model_fetcher = ModelFetcher(self.params) + self.pm = messaging.PubMaster(["modelManagerSP"]) + self.available_models: list[custom.ModelManagerSP.ModelBundle] = [] + self.selected_bundle: custom.ModelManagerSP.ModelBundle = None + self.active_bundle: custom.ModelManagerSP.ModelBundle = get_active_bundle(self.params) + self._chunk_size = 128 * 1000 # 128 KB chunks + self._download_start_times: dict[str, float] = {} # Track start time per model + + def _calculate_eta(self, filename: str, progress: float) -> int: + """Calculate ETA based on elapsed time and current progress""" + if filename not in self._download_start_times or progress <= 0: + return 60 # Default ETA for new downloads + + elapsed_time = time.monotonic() - self._download_start_times[filename] + if elapsed_time <= 0: + return 60 + + # If we're at X% after Y seconds, we can estimate total time as (Y / X) * 100 + total_estimated_time = (elapsed_time / progress) * 100 + eta = total_estimated_time - elapsed_time + + return max(1, int(eta)) # Return at least 1 second if download is ongoing + + async def _download_file(self, url: str, path: str, model) -> None: + """Downloads a file with progress tracking""" + self._download_start_times[model.fileName] = time.monotonic() + + async with aiohttp.ClientSession() as session: + async with session.get(url) as response: + response.raise_for_status() + total_size = int(response.headers.get("content-length", 0)) + bytes_downloaded = 0 + + with open(path, 'wb') as f: + async for chunk in response.content.iter_chunked(self._chunk_size): # type: bytes + f.write(chunk) + bytes_downloaded += len(chunk) + + if total_size > 0: + progress = (bytes_downloaded / total_size) * 100 + model.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.downloading + model.downloadProgress.progress = progress + model.downloadProgress.eta = self._calculate_eta(model.fileName, progress) + self._report_status() + + # Clean up start time after download completes + del self._download_start_times[model.fileName] + + async def _process_model(self, model, destination_path: str) -> None: + """Processes a single model download including verification""" + url = model.downloadUri.uri + expected_hash = model.downloadUri.sha256 + filename = model.fileName + full_path = os.path.join(destination_path, filename) + + try: + # Check existing file + if os.path.exists(full_path) and await verify_file(full_path, expected_hash): + model.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.cached + model.downloadProgress.progress = 100 + model.downloadProgress.eta = 0 + self._report_status() + return + + # Download and verify + await self._download_file(url, full_path, model) + if not await verify_file(full_path, expected_hash): + raise ValueError(f"Hash validation failed for {filename}") + + model.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.downloaded + model.downloadProgress.eta = 0 + self._report_status() + + except Exception as e: + cloudlog.error(f"Error downloading {filename}: {str(e)}") + if os.path.exists(full_path): + os.remove(full_path) + model.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.failed + model.downloadProgress.eta = 0 + self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.failed + self._report_status() + # Clean up start time if it exists + self._download_start_times.pop(model.fileName, None) + raise + + def _report_status(self) -> None: + """Reports current status through messaging system""" + msg = messaging.new_message('modelManagerSP', valid=True) + model_manager_state = msg.modelManagerSP + if self.selected_bundle: + model_manager_state.selectedBundle = self.selected_bundle + + if self.active_bundle: + model_manager_state.activeBundle = self.active_bundle + + model_manager_state.availableBundles = self.available_models + self.pm.send('modelManagerSP', msg) + + async def _download_bundle(self, model_bundle: custom.ModelManagerSP.ModelBundle, destination_path: str) -> None: + """Downloads all models in a bundle""" + self.selected_bundle = model_bundle + self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.downloading + os.makedirs(destination_path, exist_ok=True) + + try: + tasks = [self._process_model(model, destination_path) + for model in self.selected_bundle.models] + await asyncio.gather(*tasks) + self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.downloaded + self.active_bundle = self.selected_bundle + self.params.put("ModelManager_ActiveBundle", self.selected_bundle.to_bytes()) + + except Exception: + self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.failed + raise + + finally: + self._report_status() + + def download(self, model_bundle: custom.ModelManagerSP.ModelBundle, destination_path: str) -> None: + """Main entry point for downloading a model bundle""" + asyncio.run(self._download_bundle(model_bundle, destination_path)) + + def main_thread(self) -> None: + """Main thread for model management""" + rk = Ratekeeper(1, print_delay_threshold=None) + + while True: + try: + self.available_models = self.model_fetcher.get_available_models() + + if index_to_download := self.params.get("ModelManager_DownloadIndex", block=False, encoding="utf-8"): + if model_to_download := next((model for model in self.available_models if model.index == int(index_to_download)), None): + try: + self.download(model_to_download, Paths.model_root()) + except Exception as e: + cloudlog.exception(e) + finally: + self.params.put("ModelManager_DownloadIndex", "") + + self._report_status() + rk.keep_time() + + except Exception as e: + cloudlog.exception(f"Error in main thread: {str(e)}") + rk.keep_time() + + +def main(): + ModelManagerSP().main_thread() + + +if __name__ == "__main__": + main() diff --git a/sunnypilot/models/tests/__init__.py b/sunnypilot/models/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sunnypilot/models/tests/model_manager_audit.py b/sunnypilot/models/tests/model_manager_audit.py new file mode 100644 index 0000000000..1a1ea38e5b --- /dev/null +++ b/sunnypilot/models/tests/model_manager_audit.py @@ -0,0 +1,18 @@ +# Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. +# +# This file is part of sunnypilot and is licensed under the MIT License. +# See the LICENSE.md file in the root directory for more details. + +from cereal import messaging, custom + +if __name__ == "__main__": + sm = messaging.SubMaster(["modelManagerSP"]) + while True: + sm.update(500) + if sm.updated: + msg = sm["modelManagerSP"] + for model in msg.selectedBundle.models: + if model.downloadProgress.status == custom.ModelManagerSP.DownloadStatus.downloading: + print("") + print(f"{model.fileName}: {model.downloadProgress}") + print("") diff --git a/system/hardware/hw.py b/system/hardware/hw.py index dc36dc0474..a80a67d720 100644 --- a/system/hardware/hw.py +++ b/system/hardware/hw.py @@ -63,3 +63,10 @@ class Paths: if PC and platform.system() == "Darwin": return "/tmp" # This is not really shared memory on macOS, but it's the closest we can get return "/dev/shm" + + @staticmethod + def model_root() -> str: + if PC: + return str(Path(Paths.comma_home()) / "media" / "0" / "models") + else: + return "/data/media/0/models" diff --git a/system/manager/manager.py b/system/manager/manager.py index 955b5593e1..080c5c686f 100755 --- a/system/manager/manager.py +++ b/system/manager/manager.py @@ -40,6 +40,8 @@ def manager_init() -> None: ("LanguageSetting", "main_en"), ("OpenpilotEnabledToggle", "1"), ("LongitudinalPersonality", str(log.LongitudinalPersonality.standard)), + ("ModelManager_LastSyncTime", "0"), + ("ModelManager_ModelsCache", "") ] sunnypilot_default_params: list[tuple[str, str | bytes]] = [ diff --git a/system/manager/process_config.py b/system/manager/process_config.py index 09b897363c..9d94f6423a 100644 --- a/system/manager/process_config.py +++ b/system/manager/process_config.py @@ -114,6 +114,11 @@ procs = [ PythonProcess("joystick", "tools.joystick.joystick_control", and_(joystick, iscar)), ] +# sunnypilot +procs += [ + PythonProcess("models_manager", "sunnypilot.models.manager", only_offroad), +] + if os.path.exists("./github_runner.sh"): procs += [NativeProcess("github_runner_start", "system/manager", ["./github_runner.sh", "start"], and_(only_offroad, use_github_runner), sigkill=False)]