From 993c4a0d2a7b4b2b26aaf6bbda36c04064125ee3 Mon Sep 17 00:00:00 2001 From: DevTekVE Date: Sat, 4 Jan 2025 21:05:21 +0100 Subject: [PATCH] Driving Model Manager (#457) * Introduce Model Manager to handle downloads and verification This commit introduces a new Model Manager responsible for handling model downloads, including driving and navigation application models. The manager also verifies file hashes and communicates download progress for an improved user experience. The Model Manager is asynchronous and utilizes asyncio and aiohttp for enhanced performance, including robust error handling. Impacted files in the 'cereal', 'common', 'sunnypilot', and 'system' directories have been updated accordingly. The 'ModelsFetcher' process configuration has been modified to run only when off-road, ensuring optimum resource management. This update aims to enhance code clarity, improve performance, and streamline the handling of model downloads. * "Update model management and fetching for SunnyPilot" This update refactors the model management, downloading and cache verification mechanisms of SunnyPilot. New functionalities, such as smart cache handling, have been implemented in ModelFetcher in sunnypilot/models/model_fetcher.py. Also, the model downloading process has been moved to a separate async function _download_bundle in ModelManagerSP in sunnypilot/models/model_manager.py. Hash verification of files is now performed in an async function verify_file in sunnypilot/models/model_helper.py. Changes in system parameters related to model management have been reflected in system/manager/manager.py. * Integrate download ETA calculations in model manager This commit introduces a new feature that calculates and tracks the Estimated Time of Arrival (ETA) for downloading models in the model manager component. The 'eta' property in the 'DownloadProgress' structure in 'custom.capnp' is changed from 'Float32' to 'UInt32'. In the 'model_manager.py' file, a new method '_calculate_eta' has been added to perform ETA calculations. An additional dictionary '_download_start_times' has been created to keep track of the start time of each model download. The ETA is calculated every time a portion of the model file is downloaded, and it gets updated in the 'DownloadProgress' structure. Finally, the start time is cleared after the download completes. In 'model_manager_audit.py', an additional check is added to only print downloadProgress for the downloads currently in progress. * format * no default model cache {} because it can be considered a valid json, we do not want that * Refactor typing annotations to use PEP 604 syntax. Updated type hints to adopt PEP 604 union syntax (`X | None`) and replaced `List` and `Dict` with modern built-in `list` and `dict`. This change improves consistency and readability while aligning with Python 3.10+ standards. * Simplify logging messages and remove unused imports. Removed an unused import from `model_manager.py` to improve clarity and maintainability. Also refined a log message in `model_fetcher.py` by removing unnecessary formatting for consistency. * Refactor model handling and simplify cache fallback logic. Updated type annotations for `selected_bundle` in `model_manager.py` for clarity. Streamlined cache fallback logic in `model_fetcher.py` by removing redundant conditionals while preserving functionality. These changes improve code readability and maintainability. * "Fix formatting for ModelManager_DownloadIndex retrieval Condensed parameter alignment in the get method for improved readability and adherence to style guidelines. This change does not affect functionality but ensures consistent code formatting." * Need to have main defined for process_config to be able to run it * Refactor model management to support active bundle tracking Introduce the concept of an active model bundle with a new persistent parameter and API updates. Added fields for `generation` and `environment` in model metadata, improved caching, and updated methods to manage active model states efficiently. * UI commit (#515) * Refactor model management to support active bundle tracking Introduce the concept of an active model bundle with a new persistent parameter and API updates. Added fields for `generation` and `environment` in model metadata, improved caching, and updated methods to manage active model states efficiently. * Add new driving model selection feature to settings This commit introduces a new feature to the settings that allows users to select different driving models. It fetches available models and displays their download progress. The created UI also suggests a calibration reset after model download. The changes include the creation of 'SoftwarePanelSP' within 'settings.' Additionally, 'sunnypilot/SConscript' has been updated to include 'settings.cc' and 'software_panel.cc'. Changes also include localization for this feature. * Show model description during download status This update ensures the model description is displayed when a model is in the downloading state. It improves the user interface by providing real-time feedback during the download process. * Update translations for multiple languages Added new and updated translation strings in several language files, including Spanish, Arabic, Chinese (Simplified and Traditional), Turkish, Korean, Thai, Japanese, and Brazilian Portuguese. These updates include placeholder translations for new UI elements and features. * Refactor model name handling and add generation check. Replaced `bundleName` with `model_name` for better clarity in status messages. Added a generation mismatch check before showing the reset parameters dialog to avoid unnecessary prompts. * Update model handling in SoftwarePanelSP Remove unused "common/model.h" and replace "CURRENT_MODEL" with "..." as the default return value in GetModelName. Adjust logic to check for active bundles instead of selected bundles for improved accuracy. Minor text change for clarity in UI label. * Rename `GetModelName` to `GetActiveModelName` for clarity. The new name better reflects the function's purpose of retrieving the active model name, improving code readability. All relevant calls and references have been updated to ensure consistency across the codebase. * Update sunnypilot/models/model_helper.py Co-authored-by: sourcery-ai[bot] <58596630+sourcery-ai[bot]@users.noreply.github.com> * Refactor download status handling and add 'cached' state Introduce 'cached' as a new download status and adjust relevant logic across components to support it. Simplify and streamline model status handling in the software panel for better readability and maintainability. Ensure consistent status reporting for all model types. * Update translations for multiple languages Refined and expanded translations across various languages, replacing placeholders with meaningful text. This improves clarity and user experience in the multilingual interface. * Update terminology from 'bundle' to 'model' in UI texts Replaced occurrences of 'bundle' with 'model' in button labels, dialog titles, and messages in the SoftwarePanelSP code. This improves clarity and aligns terminology with current functionality. * Update translation placeholders for model fetching texts Replaced "Fetching bundles" with "Fetching models" across multiple languages to align text placeholders with the updated functionality. Adjusted related background download messages for clarity and consistency. * cleanup * not used, and likely not needed --------- Co-authored-by: sourcery-ai[bot] <58596630+sourcery-ai[bot]@users.noreply.github.com> * cleaning up * Update system/manager/process_config.py * Simplify model parsing and index handling logic. Refactored `ModelManager_DownloadIndex` retrieval to use the walrus operator, streamlining the conditional logic. Additionally, restructured model list initialization in `_parse_bundle` for improved readability and maintainability. These changes enhance code clarity and reduce redundancy. * `Improve error handling in model cache retrieval` Revised the `get` method to ensure it returns an empty dictionary on errors or missing data, avoiding potential `None`-related issues. Added logging for clearer diagnostics when cached model data is unavailable or retrieval fails. This improves reliability and debuggability of the model fetching process. * Fix cached model data handling by parsing JSON response Previously, cached model data was returned as a raw string, causing potential issues when using the data. The change ensures the cached data is properly parsed into JSON format before returning, improving reliability and consistency. * Adjust modelManagerSP rate and Ratekeeper frequency Reduced the rate for modelManagerSP in services and aligned the Ratekeeper frequency in model_manager.py to 0.1. * Update model fetcher URL and adjust modelManagerSP rate Updated the model fetcher URL to point to the correct resource for driving models. Adjusted the rate of modelManagerSP in both its service definition and the corresponding Ratekeeper initialization to 1 Hz for improved consistency. * Refactor model download logic for clarity and efficiency Simplify the logic for finding the model to download by combining redundant constructs into a single line. This improves code readability and reduces unnecessary variable assignments. * Fix cache keys for manual prebuilt actions because they were missing the cache when manually built * no need to log * formatting * revert ci changes * Refactor and restructure `modeld` to `models` module. Renamed `modeld` directory to `models` for clarity and consistency. Updated all references and imports to reflect the new structure. This improves maintainability and aligns with naming conventions. --------- Co-authored-by: sourcery-ai[bot] <58596630+sourcery-ai[bot]@users.noreply.github.com> Co-authored-by: Jason Wen --- cereal/custom.capnp | 48 ++++- cereal/log.capnp | 2 +- cereal/services.py | 3 + common/params.cc | 6 + selfdrive/ui/qt/offroad/settings.h | 2 +- selfdrive/ui/sunnypilot/SConscript | 1 + .../qt/offroad/settings/settings.cc | 3 +- .../qt/offroad/settings/software_panel.cc | 180 ++++++++++++++++++ .../qt/offroad/settings/software_panel.h | 61 ++++++ selfdrive/ui/sunnypilot/ui.cc | 1 + selfdrive/ui/translations/main_ar.ts | 75 ++++++++ selfdrive/ui/translations/main_de.ts | 75 ++++++++ selfdrive/ui/translations/main_es.ts | 75 ++++++++ selfdrive/ui/translations/main_fr.ts | 75 ++++++++ selfdrive/ui/translations/main_ja.ts | 75 ++++++++ selfdrive/ui/translations/main_ko.ts | 75 ++++++++ selfdrive/ui/translations/main_pt-BR.ts | 75 ++++++++ selfdrive/ui/translations/main_th.ts | 75 ++++++++ selfdrive/ui/translations/main_tr.ts | 75 ++++++++ selfdrive/ui/translations/main_zh-CHS.ts | 75 ++++++++ selfdrive/ui/translations/main_zh-CHT.ts | 75 ++++++++ sunnypilot/__init__.py | 0 sunnypilot/models/__init__.py | 0 sunnypilot/models/fetcher.py | 158 +++++++++++++++ sunnypilot/models/helpers.py | 32 ++++ sunnypilot/models/manager.py | 179 +++++++++++++++++ sunnypilot/models/tests/__init__.py | 0 .../models/tests/model_manager_audit.py | 18 ++ system/hardware/hw.py | 7 + system/manager/manager.py | 2 + system/manager/process_config.py | 5 + 31 files changed, 1529 insertions(+), 4 deletions(-) create mode 100644 selfdrive/ui/sunnypilot/qt/offroad/settings/software_panel.cc create mode 100644 selfdrive/ui/sunnypilot/qt/offroad/settings/software_panel.h create mode 100644 sunnypilot/__init__.py create mode 100644 sunnypilot/models/__init__.py create mode 100644 sunnypilot/models/fetcher.py create mode 100644 sunnypilot/models/helpers.py create mode 100644 sunnypilot/models/manager.py create mode 100644 sunnypilot/models/tests/__init__.py create mode 100644 sunnypilot/models/tests/model_manager_audit.py diff --git a/cereal/custom.capnp b/cereal/custom.capnp index 369222add8..70e4f742d0 100644 --- a/cereal/custom.capnp +++ b/cereal/custom.capnp @@ -11,7 +11,53 @@ $Cxx.namespace("cereal"); struct CustomReserved0 @0x81c2f05a394cf4af { } -struct CustomReserved1 @0xaedffd8f31e7b55d { +struct ModelManagerSP @0xaedffd8f31e7b55d { + activeBundle @0 :ModelBundle; + selectedBundle @1 :ModelBundle; + availableBundles @2 :List(ModelBundle); + + struct DownloadUri { + uri @0 :Text; + sha256 @1 :Text; + } + + enum Type { + drive @0; + navigation @1; + metadata @2; + } + + struct Model { + fullName @0 :Text; + fileName @1 :Text; + downloadUri @2 :DownloadUri; + downloadProgress @3 :DownloadProgress; + type @4 :Type; + } + + enum DownloadStatus { + notDownloading @0; + downloading @1; + downloaded @2; + cached @3; + failed @4; + } + + struct DownloadProgress { + status @0 :DownloadStatus; + progress @1 :Float32; + eta @2 :UInt32; + } + + struct ModelBundle { + index @0 :UInt32; + internalName @1 :Text; + displayName @2 :Text; + models @3 :List(Model); + status @4 :DownloadStatus; + generation @5 :UInt32; + environment @6 :Text; + } } struct CustomReserved2 @0xf35cc4560bbf6ec2 { diff --git a/cereal/log.capnp b/cereal/log.capnp index 68ea3099b8..96d14deca6 100644 --- a/cereal/log.capnp +++ b/cereal/log.capnp @@ -2559,7 +2559,7 @@ struct Event { # *********** Custom: reserved for forks *********** customReserved0 @107 :Custom.CustomReserved0; - customReserved1 @108 :Custom.CustomReserved1; + modelManagerSP @108 :Custom.ModelManagerSP; customReserved2 @109 :Custom.CustomReserved2; customReserved3 @110 :Custom.CustomReserved3; customReserved4 @111 :Custom.CustomReserved4; diff --git a/cereal/services.py b/cereal/services.py index 87fdca77b7..ad7bcc8700 100755 --- a/cereal/services.py +++ b/cereal/services.py @@ -74,6 +74,9 @@ _services: dict[str, tuple] = { "userFlag": (True, 0., 1), "microphone": (True, 10., 10), + # sunnypilot + "modelManagerSP": (False, 1., 1), + # debug "uiDebug": (True, 0., 1), "testJoystick": (True, 0.), diff --git a/common/params.cc b/common/params.cc index 1ab37ea84c..1e86fd27cd 100644 --- a/common/params.cc +++ b/common/params.cc @@ -200,7 +200,13 @@ std::unordered_map keys = { {"UpdaterTargetBranch", CLEAR_ON_MANAGER_START}, {"UpdaterLastFetchTime", PERSISTENT}, {"Version", PERSISTENT}, + + // sunnypilot params {"EnableGithubRunner", PERSISTENT}, + {"ModelManager_ActiveBundle", PERSISTENT}, + {"ModelManager_DownloadIndex", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_ONROAD_TRANSITION}, + {"ModelManager_LastSyncTime", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION}, + {"ModelManager_ModelsCache", PERSISTENT}, }; } // namespace diff --git a/selfdrive/ui/qt/offroad/settings.h b/selfdrive/ui/qt/offroad/settings.h index 41914c0ec0..b5b148e9a4 100644 --- a/selfdrive/ui/qt/offroad/settings.h +++ b/selfdrive/ui/qt/offroad/settings.h @@ -94,7 +94,7 @@ public: protected: void showEvent(QShowEvent *event) override; - void updateLabels(); + virtual void updateLabels(); void checkForUpdates(); bool is_onroad = false; diff --git a/selfdrive/ui/sunnypilot/SConscript b/selfdrive/ui/sunnypilot/SConscript index 5e42d8ae85..e09d6dc8af 100644 --- a/selfdrive/ui/sunnypilot/SConscript +++ b/selfdrive/ui/sunnypilot/SConscript @@ -10,6 +10,7 @@ qt_src = [ "sunnypilot/qt/window.cc", "sunnypilot/qt/home.cc", "sunnypilot/qt/offroad/settings/settings.cc", + "sunnypilot/qt/offroad/settings/software_panel.cc", "sunnypilot/qt/offroad/settings/sunnypilot_panel.cc", "sunnypilot/qt/onroad/onroad_home.cc", ] diff --git a/selfdrive/ui/sunnypilot/qt/offroad/settings/settings.cc b/selfdrive/ui/sunnypilot/qt/offroad/settings/settings.cc index ab1b0e23b7..cd7657e540 100644 --- a/selfdrive/ui/sunnypilot/qt/offroad/settings/settings.cc +++ b/selfdrive/ui/sunnypilot/qt/offroad/settings/settings.cc @@ -11,6 +11,7 @@ #include "selfdrive/ui/sunnypilot/qt/widgets/scrollview.h" #include "selfdrive/ui/qt/offroad/developer_panel.h" +#include "selfdrive/ui/sunnypilot/qt/offroad/settings/software_panel.h" #include "selfdrive/ui/sunnypilot/qt/offroad/settings/sunnypilot_panel.h" TogglesPanelSP::TogglesPanelSP(SettingsWindow *parent) : TogglesPanel(parent) { @@ -69,7 +70,7 @@ SettingsWindowSP::SettingsWindowSP(QWidget *parent) : SettingsWindow(parent) { PanelInfo(" " + tr("Device"), device, "../../sunnypilot/selfdrive/assets/offroad/icon_home.svg"), PanelInfo(" " + tr("Network"), networking, "../assets/offroad/icon_network.png"), PanelInfo(" " + tr("Toggles"), toggles, "../../sunnypilot/selfdrive/assets/offroad/icon_toggle.png"), - PanelInfo(" " + tr("Software"), new SoftwarePanel(this), "../../sunnypilot/selfdrive/assets/offroad/icon_software.png"), + PanelInfo(" " + tr("Software"), new SoftwarePanelSP(this), "../../sunnypilot/selfdrive/assets/offroad/icon_software.png"), PanelInfo(" " + tr("sunnypilot"), new SunnypilotPanel(this), "../assets/images/button_home.png"), PanelInfo(" " + tr("Developer"), new DeveloperPanel(this), "../assets/offroad/icon_shell.png"), }; diff --git a/selfdrive/ui/sunnypilot/qt/offroad/settings/software_panel.cc b/selfdrive/ui/sunnypilot/qt/offroad/settings/software_panel.cc new file mode 100644 index 0000000000..6bbef9fb94 --- /dev/null +++ b/selfdrive/ui/sunnypilot/qt/offroad/settings/software_panel.cc @@ -0,0 +1,180 @@ +/** + * Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. + * + * This file is part of sunnypilot and is licensed under the MIT License. + * See the LICENSE.md file in the root directory for more details. + */ + +#include "selfdrive/ui/sunnypilot/qt/offroad/settings/software_panel.h" + +#include +#include + +/** + * @brief Constructs the software panel with model bundle selection functionality + * @param parent Parent widget + */ +SoftwarePanelSP::SoftwarePanelSP(QWidget *parent) : SoftwarePanel(parent) { + const auto current_model = GetActiveModelName(); + currentModelLblBtn = new ButtonControlSP(tr("Current Model"), tr("SELECT"), current_model); + currentModelLblBtn->setValue(current_model); + + connect(currentModelLblBtn, &ButtonControlSP::clicked, this, &SoftwarePanelSP::handleCurrentModelLblBtnClicked); + QObject::connect(uiStateSP(), &UIStateSP::uiUpdate, this, &SoftwarePanelSP::updateLabels); + AddWidgetAt(0, currentModelLblBtn); +} + +/** + * @brief Updates the UI with bundle download progress information + * Reads status from modelManagerSP cereal message and displays status for all models + */ +void SoftwarePanelSP::handleBundleDownloadProgress() { + const SubMaster &sm = *(uiStateSP()->sm); + const auto model_manager = sm["modelManagerSP"].getModelManagerSP(); + + if (!model_manager.hasSelectedBundle()) { + currentModelLblBtn->setDescription(""); + return; + } + + const auto &bundle = model_manager.getSelectedBundle(); + const auto &models = bundle.getModels(); + QStringList status; + + // Get status for each model type in order + for (const auto &model: models) { + QString typeName; + QString modelName; + + switch (model.getType()) { + case cereal::ModelManagerSP::Type::DRIVE: + typeName = tr("Driving"); + modelName = QString::fromStdString(bundle.getDisplayName()); + break; + case cereal::ModelManagerSP::Type::NAVIGATION: + typeName = tr("Navigation"); + modelName = QString::fromStdString(model.getFullName()); + break; + case cereal::ModelManagerSP::Type::METADATA: + typeName = tr("Metadata"); + modelName = QString::fromStdString(model.getFullName()); + break; + } + + const auto &progress = model.getDownloadProgress(); + QString line; + + if (progress.getStatus() == cereal::ModelManagerSP::DownloadStatus::DOWNLOADING) { + line = tr("Downloading %1 model [%2]... (%3%)").arg(typeName, modelName).arg(progress.getProgress(), 0, 'f', 2); + } else if (progress.getStatus() == cereal::ModelManagerSP::DownloadStatus::DOWNLOADED) { + line = tr("%1 model [%2] downloaded").arg(typeName, modelName); + } else if (progress.getStatus() == cereal::ModelManagerSP::DownloadStatus::CACHED) { + line = tr("%1 model [%2] from cache").arg(typeName, modelName); + } else if (progress.getStatus() == cereal::ModelManagerSP::DownloadStatus::FAILED) { + line = tr("%1 model [%2] download failed").arg(typeName, modelName); + } else { + line = tr("%1 model [%2] pending...").arg(typeName, modelName); + } + status.append(line); + } + + currentModelLblBtn->setDescription(status.join("\n")); + + if (bundle.getStatus() == cereal::ModelManagerSP::DownloadStatus::DOWNLOADING) { + currentModelLblBtn->showDescription(); + } + + currentModelLblBtn->setEnabled(!is_onroad && !isDownloading()); +} + +/** + * @brief Gets the name of the currently selected model bundle + * @return Display name of the selected bundle or default model name + */ +QString SoftwarePanelSP::GetActiveModelName() { + const SubMaster &sm = *(uiStateSP()->sm); + const auto model_manager = sm["modelManagerSP"].getModelManagerSP(); + + if (model_manager.hasActiveBundle()) { + return QString::fromStdString(model_manager.getActiveBundle().getDisplayName()); + } + + return ""; +} + +/** + * @brief Handles the model bundle selection button click + * Displays available bundles, allows selection, and initiates download + */ +void SoftwarePanelSP::handleCurrentModelLblBtnClicked() { + currentModelLblBtn->setEnabled(false); + currentModelLblBtn->setValue(tr("Fetching models...")); + + const SubMaster &sm = *(uiStateSP()->sm); + const auto model_manager = sm["modelManagerSP"].getModelManagerSP(); + + // Create mapping of bundle indices to display names + QMap index_to_bundle; + const auto bundles = model_manager.getAvailableBundles(); + for (const auto &bundle: bundles) { + index_to_bundle.insert(bundle.getIndex(), QString::fromStdString(bundle.getDisplayName())); + } + + // Sort bundles by index in descending order + QStringList bundleNames; + auto indices = index_to_bundle.keys(); + std::sort(indices.begin(), indices.end(), std::greater()); + for (const auto &index: indices) { + bundleNames.append(index_to_bundle[index]); + } + + currentModelLblBtn->setEnabled(!is_onroad); + currentModelLblBtn->setValue(GetActiveModelName()); + + const QString selectedBundleName = MultiOptionDialog::getSelection( + tr("Select a Model"), bundleNames, GetActiveModelName(), this); + + if (selectedBundleName.isEmpty() || !canContinueOnMeteredDialog()) { + return; + } + + // Find selected bundle and initiate download + for (const auto &bundle: bundles) { + if (QString::fromStdString(bundle.getDisplayName()) == selectedBundleName) { + params.put("ModelManager_DownloadIndex", std::to_string(bundle.getIndex())); + if (bundle.getGeneration() != model_manager.getActiveBundle().getGeneration()) { + showResetParamsDialog(); + } + break; + } + } + + updateLabels(); +} + +/** + * @brief Updates the UI elements based on current state + */ +void SoftwarePanelSP::updateLabels() { + if (!isVisible()) { + return; + } + + handleBundleDownloadProgress(); + currentModelLblBtn->setValue(GetActiveModelName()); + SoftwarePanel::updateLabels(); +} + +/** + * @brief Shows dialog prompting user to reset calibration after model download + */ +void SoftwarePanelSP::showResetParamsDialog() { + const auto confirmMsg = tr("Model download has started in the background.") + "\n" + + tr("We STRONGLY suggest you to reset calibration. Would you like to do that now?"); + const auto button_text = tr("Reset Calibration"); + + if (showConfirmationDialog(confirmMsg, button_text, false)) { + params.remove("CalibrationParams"); + params.remove("LiveTorqueParameters"); + } +} diff --git a/selfdrive/ui/sunnypilot/qt/offroad/settings/software_panel.h b/selfdrive/ui/sunnypilot/qt/offroad/settings/software_panel.h new file mode 100644 index 0000000000..fd837e31fa --- /dev/null +++ b/selfdrive/ui/sunnypilot/qt/offroad/settings/software_panel.h @@ -0,0 +1,61 @@ +/** + * Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. + * + * This file is part of sunnypilot and is licensed under the MIT License. + * See the LICENSE.md file in the root directory for more details. + */ + +#pragma once + +#include +#include "selfdrive/ui/sunnypilot/ui.h" +#include "selfdrive/ui/qt/offroad/settings.h" + +class SoftwarePanelSP final : public SoftwarePanel { + Q_OBJECT + +public: + explicit SoftwarePanelSP(QWidget *parent = nullptr); + +private: + QString GetActiveModelName(); + + bool isDownloading() const { + const SubMaster &sm = *(uiStateSP()->sm); + const auto model_manager = sm["modelManagerSP"].getModelManagerSP(); + + if (!model_manager.hasSelectedBundle()) { + return false; + } + + const auto &selected_bundle = model_manager.getSelectedBundle(); + return selected_bundle.getStatus() == cereal::ModelManagerSP::DownloadStatus::DOWNLOADING; + } + + // UI update related methods + void updateLabels() override; + void handleCurrentModelLblBtnClicked(); + void handleBundleDownloadProgress(); + void showResetParamsDialog(); + + bool canContinueOnMeteredDialog() { + if (!is_metered) return true; + return showConfirmationDialog(QString(), QString(), is_metered); + } + + inline bool showConfirmationDialog(const QString &message = QString(), const QString &confirmButtonText = QString(), const bool show_metered_warning = false) { + return showConfirmationDialog(this, message, confirmButtonText, show_metered_warning); + } + + static inline bool showConfirmationDialog(QWidget *parent, const QString &message = QString(), const QString &confirmButtonText = QString(), const bool show_metered_warning = false) { + const QString warning_message = show_metered_warning ? tr("Warning: You are on a metered connection!") : QString(); + const QString final_message = QString("%1%2").arg(!message.isEmpty() ? message + "\n" : QString(), warning_message); + const QString final_buttonText = !confirmButtonText.isEmpty() ? confirmButtonText : QString(tr("Continue") + " %1").arg(show_metered_warning ? tr("on Metered") : ""); + + return ConfirmationDialog::confirm(final_message, final_buttonText, parent); + } + + bool is_metered{}; + bool is_wifi{}; + ButtonControlSP *currentModelLblBtn; +}; diff --git a/selfdrive/ui/sunnypilot/ui.cc b/selfdrive/ui/sunnypilot/ui.cc index e869d3b2dd..cb87743735 100644 --- a/selfdrive/ui/sunnypilot/ui.cc +++ b/selfdrive/ui/sunnypilot/ui.cc @@ -18,6 +18,7 @@ UIStateSP::UIStateSP(QObject *parent) : UIState(parent) { "modelV2", "controlsState", "liveCalibration", "radarState", "deviceState", "pandaStates", "carParams", "driverMonitoringState", "carState", "driverStateV2", "wideRoadCameraState", "managerState", "selfdriveState", "longitudinalPlan", + "modelManagerSP", }); // update timer diff --git a/selfdrive/ui/translations/main_ar.ts b/selfdrive/ui/translations/main_ar.ts index ac5433ff47..6845b9c7db 100644 --- a/selfdrive/ui/translations/main_ar.ts +++ b/selfdrive/ui/translations/main_ar.ts @@ -944,6 +944,81 @@ This may take up to a minute. أحدث نسخة، آخر تحقق %1 + + SoftwarePanelSP + + SELECT + اختيار + + + We STRONGLY suggest you to reset calibration. Would you like to do that now? + + + + Reset Calibration + إعادة ضبط المعايرة + + + Warning: You are on a metered connection! + + + + Continue + متابعة + + + on Metered + + + + %1 model [%2] pending... + + + + Current Model + + + + Driving + + + + Navigation + + + + Metadata + + + + Downloading %1 model [%2]... (%3%) + + + + %1 model [%2] downloaded + + + + %1 model [%2] download failed + + + + %1 model [%2] from cache + + + + Select a Model + + + + Fetching models... + + + + Model download has started in the background. + + + SshControl diff --git a/selfdrive/ui/translations/main_de.ts b/selfdrive/ui/translations/main_de.ts index 8dbc436792..7db919228b 100644 --- a/selfdrive/ui/translations/main_de.ts +++ b/selfdrive/ui/translations/main_de.ts @@ -928,6 +928,81 @@ This may take up to a minute. + + SoftwarePanelSP + + SELECT + AUSWÄHLEN + + + We STRONGLY suggest you to reset calibration. Would you like to do that now? + + + + Reset Calibration + Neu kalibrieren + + + Warning: You are on a metered connection! + + + + Continue + Fortsetzen + + + on Metered + + + + %1 model [%2] pending... + + + + Current Model + + + + Driving + + + + Navigation + + + + Metadata + + + + Downloading %1 model [%2]... (%3%) + + + + %1 model [%2] downloaded + + + + %1 model [%2] download failed + + + + %1 model [%2] from cache + + + + Select a Model + + + + Fetching models... + + + + Model download has started in the background. + + + SshControl diff --git a/selfdrive/ui/translations/main_es.ts b/selfdrive/ui/translations/main_es.ts index 5f7e980aa0..5d19df390d 100644 --- a/selfdrive/ui/translations/main_es.ts +++ b/selfdrive/ui/translations/main_es.ts @@ -928,6 +928,81 @@ Esto puede tardar un minuto. actualizado, último chequeo %1 + + SoftwarePanelSP + + SELECT + SELECCIONAR + + + We STRONGLY suggest you to reset calibration. Would you like to do that now? + + + + Reset Calibration + Formatear Calibración + + + Warning: You are on a metered connection! + + + + Continue + Continuar + + + on Metered + + + + %1 model [%2] pending... + modelo de %1 [%2] pendiente... + + + Fetching models... + Obteniendo modelos + + + Model download has started in the background. + Descarga de modelo iniciada en segundo plano. + + + Current Model + Modelo Actual + + + Driving + Conducción + + + Navigation + Navegación + + + Metadata + Metadatos + + + Downloading %1 model [%2]... (%3%) + Descargando modelo de %1 [%2]... (%3%) + + + %1 model [%2] downloaded + Modelo de %1 [%2] descargado + + + %1 model [%2] download failed + Falló descarga modelo de %1 [%2] + + + %1 model [%2] from cache + Modelo de %1 [%2] desde caché + + + Select a Model + Selecciona un Modelo + + SshControl diff --git a/selfdrive/ui/translations/main_fr.ts b/selfdrive/ui/translations/main_fr.ts index 2348b10199..58713533d7 100644 --- a/selfdrive/ui/translations/main_fr.ts +++ b/selfdrive/ui/translations/main_fr.ts @@ -928,6 +928,81 @@ Cela peut prendre jusqu'à une minute. à jour, dernière vérification %1 + + SoftwarePanelSP + + SELECT + SÉLECTIONNER + + + We STRONGLY suggest you to reset calibration. Would you like to do that now? + + + + Reset Calibration + Réinitialiser la calibration + + + Warning: You are on a metered connection! + + + + Continue + Continuer + + + on Metered + + + + %1 model [%2] pending... + + + + Current Model + + + + Driving + + + + Navigation + + + + Metadata + + + + Downloading %1 model [%2]... (%3%) + + + + %1 model [%2] downloaded + + + + %1 model [%2] download failed + + + + %1 model [%2] from cache + + + + Select a Model + + + + Fetching models... + + + + Model download has started in the background. + + + SshControl diff --git a/selfdrive/ui/translations/main_ja.ts b/selfdrive/ui/translations/main_ja.ts index 68c629a011..a916b34b84 100644 --- a/selfdrive/ui/translations/main_ja.ts +++ b/selfdrive/ui/translations/main_ja.ts @@ -922,6 +922,81 @@ This may take up to a minute. + + SoftwarePanelSP + + SELECT + 選択 + + + We STRONGLY suggest you to reset calibration. Would you like to do that now? + + + + Reset Calibration + ｷｬﾘﾌﾞﾚｰｼｮﾝをﾘｾｯﾄ + + + Warning: You are on a metered connection! + + + + Continue + 続ける + + + on Metered + + + + %1 model [%2] pending... + + + + Current Model + + + + Driving + + + + Navigation + + + + Metadata + + + + Downloading %1 model [%2]... (%3%) + + + + %1 model [%2] downloaded + + + + %1 model [%2] download failed + + + + %1 model [%2] from cache + + + + Select a Model + + + + Fetching models... + + + + Model download has started in the background. + + + SshControl diff --git a/selfdrive/ui/translations/main_ko.ts b/selfdrive/ui/translations/main_ko.ts index f26ad2d624..c9a7d8981e 100644 --- a/selfdrive/ui/translations/main_ko.ts +++ b/selfdrive/ui/translations/main_ko.ts @@ -924,6 +924,81 @@ This may take up to a minute. 업데이트 안함 + + SoftwarePanelSP + + SELECT + 선택 + + + We STRONGLY suggest you to reset calibration. Would you like to do that now? + + + + Reset Calibration + 캘리브레이션 초기화 + + + Warning: You are on a metered connection! + + + + Continue + 계속 + + + on Metered + + + + %1 model [%2] pending... + + + + Current Model + + + + Driving + + + + Navigation + + + + Metadata + + + + Downloading %1 model [%2]... (%3%) + + + + %1 model [%2] downloaded + + + + %1 model [%2] download failed + + + + %1 model [%2] from cache + + + + Select a Model + + + + Fetching models... + + + + Model download has started in the background. + + + SshControl diff --git a/selfdrive/ui/translations/main_pt-BR.ts b/selfdrive/ui/translations/main_pt-BR.ts index be5732275d..e6cde1181c 100644 --- a/selfdrive/ui/translations/main_pt-BR.ts +++ b/selfdrive/ui/translations/main_pt-BR.ts @@ -928,6 +928,81 @@ Isso pode levar até um minuto. nunca + + SoftwarePanelSP + + SELECT + SELECIONE + + + We STRONGLY suggest you to reset calibration. Would you like to do that now? + + + + Reset Calibration + Reinicializar Calibragem + + + Warning: You are on a metered connection! + + + + Continue + Continuar + + + on Metered + + + + %1 model [%2] pending... + + + + Current Model + + + + Driving + + + + Navigation + + + + Metadata + + + + Downloading %1 model [%2]... (%3%) + + + + %1 model [%2] downloaded + + + + %1 model [%2] download failed + + + + %1 model [%2] from cache + + + + Select a Model + + + + Fetching models... + + + + Model download has started in the background. + + + SshControl diff --git a/selfdrive/ui/translations/main_th.ts b/selfdrive/ui/translations/main_th.ts index fc8bacd8be..41b9c1e53e 100644 --- a/selfdrive/ui/translations/main_th.ts +++ b/selfdrive/ui/translations/main_th.ts @@ -924,6 +924,81 @@ This may take up to a minute. ล่าสุดแล้ว ตรวจสอบครั้งสุดท้ายเมื่อ %1 + + SoftwarePanelSP + + SELECT + เลือก + + + We STRONGLY suggest you to reset calibration. Would you like to do that now? + + + + Reset Calibration + รีเซ็ตการคาลิเบรท + + + Warning: You are on a metered connection! + + + + Continue + ดำเนินการต่อ + + + on Metered + + + + %1 model [%2] pending... + + + + Current Model + + + + Driving + + + + Navigation + + + + Metadata + + + + Downloading %1 model [%2]... (%3%) + + + + %1 model [%2] downloaded + + + + %1 model [%2] download failed + + + + %1 model [%2] from cache + + + + Select a Model + + + + Fetching models... + + + + Model download has started in the background. + + + SshControl diff --git a/selfdrive/ui/translations/main_tr.ts b/selfdrive/ui/translations/main_tr.ts index d89e3c6693..0927aa6279 100644 --- a/selfdrive/ui/translations/main_tr.ts +++ b/selfdrive/ui/translations/main_tr.ts @@ -922,6 +922,81 @@ This may take up to a minute. + + SoftwarePanelSP + + SELECT + + + + We STRONGLY suggest you to reset calibration. Would you like to do that now? + + + + Reset Calibration + Kalibrasyonu sıfırla + + + Warning: You are on a metered connection! + + + + Continue + Devam et + + + on Metered + + + + %1 model [%2] pending... + + + + Current Model + + + + Driving + + + + Navigation + + + + Metadata + + + + Downloading %1 model [%2]... (%3%) + + + + %1 model [%2] downloaded + + + + %1 model [%2] download failed + + + + %1 model [%2] from cache + + + + Select a Model + + + + Fetching models... + + + + Model download has started in the background. + + + SshControl diff --git a/selfdrive/ui/translations/main_zh-CHS.ts b/selfdrive/ui/translations/main_zh-CHS.ts index 207d1c8b33..603396ba67 100644 --- a/selfdrive/ui/translations/main_zh-CHS.ts +++ b/selfdrive/ui/translations/main_zh-CHS.ts @@ -924,6 +924,81 @@ This may take up to a minute. 从未更新 + + SoftwarePanelSP + + SELECT + 选择 + + + We STRONGLY suggest you to reset calibration. Would you like to do that now? + + + + Reset Calibration + 重置设备校准 + + + Warning: You are on a metered connection! + + + + Continue + 继续 + + + on Metered + + + + %1 model [%2] pending... + + + + Current Model + + + + Driving + + + + Navigation + + + + Metadata + + + + Downloading %1 model [%2]... (%3%) + + + + %1 model [%2] downloaded + + + + %1 model [%2] download failed + + + + %1 model [%2] from cache + + + + Select a Model + + + + Fetching models... + + + + Model download has started in the background. + + + SshControl diff --git a/selfdrive/ui/translations/main_zh-CHT.ts b/selfdrive/ui/translations/main_zh-CHT.ts index c820d85333..99a0e05349 100644 --- a/selfdrive/ui/translations/main_zh-CHT.ts +++ b/selfdrive/ui/translations/main_zh-CHT.ts @@ -924,6 +924,81 @@ This may take up to a minute. 從未更新 + + SoftwarePanelSP + + SELECT + 選取 + + + We STRONGLY suggest you to reset calibration. Would you like to do that now? + + + + Reset Calibration + 重設校準 + + + Warning: You are on a metered connection! + + + + Continue + 繼續 + + + on Metered + + + + %1 model [%2] pending... + + + + Current Model + + + + Driving + + + + Navigation + + + + Metadata + + + + Downloading %1 model [%2]... (%3%) + + + + %1 model [%2] downloaded + + + + %1 model [%2] download failed + + + + %1 model [%2] from cache + + + + Select a Model + + + + Fetching models... + + + + Model download has started in the background. + + + SshControl diff --git a/sunnypilot/__init__.py b/sunnypilot/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sunnypilot/models/__init__.py b/sunnypilot/models/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sunnypilot/models/fetcher.py b/sunnypilot/models/fetcher.py new file mode 100644 index 0000000000..67f7efbae9 --- /dev/null +++ b/sunnypilot/models/fetcher.py @@ -0,0 +1,158 @@ +# Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. +# +# This file is part of sunnypilot and is licensed under the MIT License. +# See the LICENSE.md file in the root directory for more details. + +import json +import time + +import requests +from openpilot.common.params import Params +from openpilot.common.swaglog import cloudlog + +from cereal import custom + + +class ModelParser: + """Handles parsing of model data into cereal objects""" + + @staticmethod + def _parse_model(full_name: str, file_name: str, uri_data: dict, + model_type: custom.ModelManagerSP.Type) -> custom.ModelManagerSP.Model: + model = custom.ModelManagerSP.Model() + download_uri = custom.ModelManagerSP.DownloadUri() + + download_uri.uri = uri_data["url"] + download_uri.sha256 = uri_data["sha256"] + + model.fullName = full_name + model.fileName = file_name + model.downloadUri = download_uri + model.type = model_type + + return model + + @staticmethod + def _parse_bundle(key: str, value: dict) -> custom.ModelManagerSP.ModelBundle: + model_bundle = custom.ModelManagerSP.ModelBundle() + + # Parse main driving model + models = [ + ModelParser._parse_model( + value["full_name"], + value["file_name"], + value["download_uri"], + custom.ModelManagerSP.Type.drive + ) + ] + + # Parse navigation model if exists + if value.get("download_uri_nav"): + models.append(ModelParser._parse_model( + value["full_name_nav"], + value["file_name_nav"], + value["download_uri_nav"], + custom.ModelManagerSP.Type.navigation + )) + + # Parse metadata model if exists + if value.get("download_uri_metadata"): + models.append(ModelParser._parse_model( + value["full_name_metadata"], + value["file_name_metadata"], + value["download_uri_metadata"], + custom.ModelManagerSP.Type.metadata + )) + + model_bundle.index = int(value["index"]) + model_bundle.internalName = key + model_bundle.displayName = value["display_name"] + model_bundle.models = models + model_bundle.status = 0 + model_bundle.generation = int(value["generation"]) + model_bundle.environment = value["environment"] + + return model_bundle + + @staticmethod + def parse_models(json_data: dict) -> list[custom.ModelManagerSP.ModelBundle]: + return [ModelParser._parse_bundle(key, value) for key, value in json_data.items()] + + +class ModelCache: + """Handles caching of model data to avoid frequent remote fetches""" + + def __init__(self, params: Params, cache_timeout: int = int(3600 * 1e9)): + self.params = params + self.cache_timeout = cache_timeout + self._LAST_SYNC_KEY = "ModelManager_LastSyncTime" + self._CACHE_KEY = "ModelManager_ModelsCache" + + def _is_expired(self) -> bool: + """Checks if the cache has expired""" + current_time = int(time.monotonic() * 1e9) + last_sync = int(self.params.get(self._LAST_SYNC_KEY, encoding="utf-8") or 0) + return (current_time - last_sync) >= self.cache_timeout + + def get(self) -> tuple[dict, bool]: + """ + Retrieves cached model data and expiration status atomically. + Returns: Tuple of (cached_data, is_expired) + If no cached data exists or on error, returns an empty dict + """ + try: + cached_data = self.params.get(self._CACHE_KEY, encoding="utf-8") + if not cached_data: + cloudlog.warning("No cached model data available") + return {}, True + return json.loads(cached_data), self._is_expired() + except Exception as e: + cloudlog.exception(f"Error retrieving cached model data: {str(e)}") + return {}, True + + def set(self, data: dict) -> None: + """Updates the cache with new model data""" + self.params.put(self._CACHE_KEY, json.dumps(data)) + self.params.put(self._LAST_SYNC_KEY, str(int(time.monotonic() * 1e9))) + + +class ModelFetcher: + """Handles fetching and caching of model data from remote source""" + MODEL_URL = "https://docs.sunnypilot.ai/driving_models.json" + + def __init__(self, params: Params): + self.params = params + self.model_cache = ModelCache(params) + self.model_parser = ModelParser() + + def _fetch_and_cache_models(self) -> list[custom.ModelManagerSP.ModelBundle]: + """Fetches fresh model data from remote and updates cache""" + try: + response = requests.get(self.MODEL_URL, timeout=10) + response.raise_for_status() + json_data = response.json() + + self.model_cache.set(json_data) + cloudlog.debug("Successfully updated models cache") + return self.model_parser.parse_models(json_data) + except Exception: + cloudlog.exception("Error fetching models") + raise + + def get_available_models(self) -> list[custom.ModelManagerSP.ModelBundle]: + """Gets the list of available models, with smart cache handling""" + cached_data, is_expired = self.model_cache.get() + + if cached_data and not is_expired: + cloudlog.debug("Using valid cached models data") + return self.model_parser.parse_models(cached_data) + + try: + return self._fetch_and_cache_models() + except Exception: + if not cached_data: + cloudlog.exception("Failed to fetch fresh data and no cache available") + raise + + cloudlog.warning("Failed to fetch fresh data. Using expired cache as fallback") + return self.model_parser.parse_models(cached_data) diff --git a/sunnypilot/models/helpers.py b/sunnypilot/models/helpers.py new file mode 100644 index 0000000000..fe6ac2133c --- /dev/null +++ b/sunnypilot/models/helpers.py @@ -0,0 +1,32 @@ +# Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. +# +# This file is part of sunnypilot and is licensed under the MIT License. +# See the LICENSE.md file in the root directory for more details. + +import hashlib +import os +from openpilot.common.params import Params +from cereal import custom, messaging + + +async def verify_file(file_path: str, expected_hash: str) -> bool: + """Verifies file hash against expected hash""" + if not os.path.exists(file_path): + return False + + sha256_hash = hashlib.sha256() + with open(file_path, "rb") as file: + for chunk in iter(lambda: file.read(4096), b""): + sha256_hash.update(chunk) + + return sha256_hash.hexdigest().lower() == expected_hash.lower() + +def get_active_bundle(params: Params) -> custom.ModelManagerSP.ModelBundle: + """Gets the active model bundle from cache""" + if params is None: + params = Params() + + if active_bundle := params.get("ModelManager_ActiveBundle"): + return messaging.log_from_bytes(active_bundle, custom.ModelManagerSP.ModelBundle) + + return None diff --git a/sunnypilot/models/manager.py b/sunnypilot/models/manager.py new file mode 100644 index 0000000000..4226c6c679 --- /dev/null +++ b/sunnypilot/models/manager.py @@ -0,0 +1,179 @@ +# Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. +# +# This file is part of sunnypilot and is licensed under the MIT License. +# See the LICENSE.md file in the root directory for more details. + +import asyncio +import os +import time + +import aiohttp +from openpilot.common.params import Params +from openpilot.common.realtime import Ratekeeper +from openpilot.common.swaglog import cloudlog +from openpilot.system.hardware.hw import Paths + +from cereal import messaging, custom +from sunnypilot.models.fetcher import ModelFetcher +from sunnypilot.models.helpers import verify_file, get_active_bundle + + +class ModelManagerSP: + """Manages model downloads and status reporting""" + + def __init__(self): + self.params = Params() + self.model_fetcher = ModelFetcher(self.params) + self.pm = messaging.PubMaster(["modelManagerSP"]) + self.available_models: list[custom.ModelManagerSP.ModelBundle] = [] + self.selected_bundle: custom.ModelManagerSP.ModelBundle = None + self.active_bundle: custom.ModelManagerSP.ModelBundle = get_active_bundle(self.params) + self._chunk_size = 128 * 1000 # 128 KB chunks + self._download_start_times: dict[str, float] = {} # Track start time per model + + def _calculate_eta(self, filename: str, progress: float) -> int: + """Calculate ETA based on elapsed time and current progress""" + if filename not in self._download_start_times or progress <= 0: + return 60 # Default ETA for new downloads + + elapsed_time = time.monotonic() - self._download_start_times[filename] + if elapsed_time <= 0: + return 60 + + # If we're at X% after Y seconds, we can estimate total time as (Y / X) * 100 + total_estimated_time = (elapsed_time / progress) * 100 + eta = total_estimated_time - elapsed_time + + return max(1, int(eta)) # Return at least 1 second if download is ongoing + + async def _download_file(self, url: str, path: str, model) -> None: + """Downloads a file with progress tracking""" + self._download_start_times[model.fileName] = time.monotonic() + + async with aiohttp.ClientSession() as session: + async with session.get(url) as response: + response.raise_for_status() + total_size = int(response.headers.get("content-length", 0)) + bytes_downloaded = 0 + + with open(path, 'wb') as f: + async for chunk in response.content.iter_chunked(self._chunk_size): # type: bytes + f.write(chunk) + bytes_downloaded += len(chunk) + + if total_size > 0: + progress = (bytes_downloaded / total_size) * 100 + model.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.downloading + model.downloadProgress.progress = progress + model.downloadProgress.eta = self._calculate_eta(model.fileName, progress) + self._report_status() + + # Clean up start time after download completes + del self._download_start_times[model.fileName] + + async def _process_model(self, model, destination_path: str) -> None: + """Processes a single model download including verification""" + url = model.downloadUri.uri + expected_hash = model.downloadUri.sha256 + filename = model.fileName + full_path = os.path.join(destination_path, filename) + + try: + # Check existing file + if os.path.exists(full_path) and await verify_file(full_path, expected_hash): + model.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.cached + model.downloadProgress.progress = 100 + model.downloadProgress.eta = 0 + self._report_status() + return + + # Download and verify + await self._download_file(url, full_path, model) + if not await verify_file(full_path, expected_hash): + raise ValueError(f"Hash validation failed for {filename}") + + model.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.downloaded + model.downloadProgress.eta = 0 + self._report_status() + + except Exception as e: + cloudlog.error(f"Error downloading {filename}: {str(e)}") + if os.path.exists(full_path): + os.remove(full_path) + model.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.failed + model.downloadProgress.eta = 0 + self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.failed + self._report_status() + # Clean up start time if it exists + self._download_start_times.pop(model.fileName, None) + raise + + def _report_status(self) -> None: + """Reports current status through messaging system""" + msg = messaging.new_message('modelManagerSP', valid=True) + model_manager_state = msg.modelManagerSP + if self.selected_bundle: + model_manager_state.selectedBundle = self.selected_bundle + + if self.active_bundle: + model_manager_state.activeBundle = self.active_bundle + + model_manager_state.availableBundles = self.available_models + self.pm.send('modelManagerSP', msg) + + async def _download_bundle(self, model_bundle: custom.ModelManagerSP.ModelBundle, destination_path: str) -> None: + """Downloads all models in a bundle""" + self.selected_bundle = model_bundle + self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.downloading + os.makedirs(destination_path, exist_ok=True) + + try: + tasks = [self._process_model(model, destination_path) + for model in self.selected_bundle.models] + await asyncio.gather(*tasks) + self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.downloaded + self.active_bundle = self.selected_bundle + self.params.put("ModelManager_ActiveBundle", self.selected_bundle.to_bytes()) + + except Exception: + self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.failed + raise + + finally: + self._report_status() + + def download(self, model_bundle: custom.ModelManagerSP.ModelBundle, destination_path: str) -> None: + """Main entry point for downloading a model bundle""" + asyncio.run(self._download_bundle(model_bundle, destination_path)) + + def main_thread(self) -> None: + """Main thread for model management""" + rk = Ratekeeper(1, print_delay_threshold=None) + + while True: + try: + self.available_models = self.model_fetcher.get_available_models() + + if index_to_download := self.params.get("ModelManager_DownloadIndex", block=False, encoding="utf-8"): + if model_to_download := next((model for model in self.available_models if model.index == int(index_to_download)), None): + try: + self.download(model_to_download, Paths.model_root()) + except Exception as e: + cloudlog.exception(e) + finally: + self.params.put("ModelManager_DownloadIndex", "") + + self._report_status() + rk.keep_time() + + except Exception as e: + cloudlog.exception(f"Error in main thread: {str(e)}") + rk.keep_time() + + +def main(): + ModelManagerSP().main_thread() + + +if __name__ == "__main__": + main() diff --git a/sunnypilot/models/tests/__init__.py b/sunnypilot/models/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sunnypilot/models/tests/model_manager_audit.py b/sunnypilot/models/tests/model_manager_audit.py new file mode 100644 index 0000000000..1a1ea38e5b --- /dev/null +++ b/sunnypilot/models/tests/model_manager_audit.py @@ -0,0 +1,18 @@ +# Copyright (c) 2021-, Haibin Wen, sunnypilot, and a number of other contributors. +# +# This file is part of sunnypilot and is licensed under the MIT License. +# See the LICENSE.md file in the root directory for more details. + +from cereal import messaging, custom + +if __name__ == "__main__": + sm = messaging.SubMaster(["modelManagerSP"]) + while True: + sm.update(500) + if sm.updated: + msg = sm["modelManagerSP"] + for model in msg.selectedBundle.models: + if model.downloadProgress.status == custom.ModelManagerSP.DownloadStatus.downloading: + print("") + print(f"{model.fileName}: {model.downloadProgress}") + print("") diff --git a/system/hardware/hw.py b/system/hardware/hw.py index dc36dc0474..a80a67d720 100644 --- a/system/hardware/hw.py +++ b/system/hardware/hw.py @@ -63,3 +63,10 @@ class Paths: if PC and platform.system() == "Darwin": return "/tmp" # This is not really shared memory on macOS, but it's the closest we can get return "/dev/shm" + + @staticmethod + def model_root() -> str: + if PC: + return str(Path(Paths.comma_home()) / "media" / "0" / "models") + else: + return "/data/media/0/models" diff --git a/system/manager/manager.py b/system/manager/manager.py index 4a5da353e9..8afb294971 100755 --- a/system/manager/manager.py +++ b/system/manager/manager.py @@ -40,6 +40,8 @@ def manager_init() -> None: ("LanguageSetting", "main_en"), ("OpenpilotEnabledToggle", "1"), ("LongitudinalPersonality", str(log.LongitudinalPersonality.standard)), + ("ModelManager_LastSyncTime", "0"), + ("ModelManager_ModelsCache", "") ] if params.get_bool("RecordFrontLock"): diff --git a/system/manager/process_config.py b/system/manager/process_config.py index 09b897363c..9d94f6423a 100644 --- a/system/manager/process_config.py +++ b/system/manager/process_config.py @@ -114,6 +114,11 @@ procs = [ PythonProcess("joystick", "tools.joystick.joystick_control", and_(joystick, iscar)), ] +# sunnypilot +procs += [ + PythonProcess("models_manager", "sunnypilot.models.manager", only_offroad), +] + if os.path.exists("./github_runner.sh"): procs += [NativeProcess("github_runner_start", "system/manager", ["./github_runner.sh", "start"], and_(only_offroad, use_github_runner), sigkill=False)]