bugfix: fix fetching params for sunnylink and backup (#1177)

* Hotfix for the params stuff until I rework this properly and leverage the new data types

* Revert "Hotfix for the params stuff until I rework this properly and leverage the new data types"

This reverts commit c6031b29d92d3ff5b679ffce3ba53611bb2dba0e.

* refactor: enhance getParams function to support JSON and bytes types with optional compression

* refactor: add TODO for enhancing server support of metadata in sunnylinkd.py

* lint and clean

* refactor: update value handling in getParams to return decoded values for JSON serialization

* refactor: simplify params_dict initialization by removing type hint

* refactor: update response handling in getParams to include JSON serialization of params

* refactor: update response handling in getParams to include JSON serialization of params

* Add to dic types

* refactor: extract get_param_as_byte function for improved parameter handling and fix backup inconsistencies

* cleanup

* ensure error propagates on backup fail

sunnypilot/sunnylink/athena/sunnylinkd.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import base64
 import errno
 import gzip
+import json
 import os
 import ssl
 import threading
@@ -22,7 +23,7 @@ from websocket import (ABNF, WebSocket, WebSocketException, WebSocketTimeoutExce
 
 import cereal.messaging as messaging
 from sunnypilot.sunnylink.api import SunnylinkApi
-from sunnypilot.sunnylink.utils import sunnylink_need_register, sunnylink_ready
+from sunnypilot.sunnylink.utils import sunnylink_need_register, sunnylink_ready, get_param_as_byte
 
 SUNNYLINK_ATHENA_HOST = os.getenv('SUNNYLINK_ATHENA_HOST', 'wss://ws.stg.api.sunnypilot.ai')
 HANDLER_THREADS = int(os.getenv('HANDLER_THREADS', "4"))
@@ -179,16 +180,22 @@ def getParamsAllKeys() -> list[str]:
 
 @dispatcher.add_method
 def getParams(params_keys: list[str], compression: bool = False) -> str | dict[str, str]:
+  params = Params()
+
   try:
-    params = Params()
-    params_dict: dict[str, bytes] = {key: params.get(key) or b'' for key in params_keys}
+    param_keys_validated = [key for key in params_keys if key in getParamsAllKeys()]
+    params_dict:  dict[str, list[dict[str, str | bool | int ]]] = {"params": [
+      {
+        "key": key,
+        "value": base64.b64encode(gzip.compress(get_param_as_byte(key)) if compression else get_param_as_byte(key)).decode('utf-8'),
+        "type": int(params.get_type(key).value),
+        "is_compressed": compression
+      } for key in param_keys_validated
+    ]}
 
-    # Compress the values before encoding to base64 as output from params.get is bytes and same for compression
-    if compression:
-      params_dict = {key: gzip.compress(value) for key, value in params_dict.items()}
-
-    # Last step is to encode the values to base64 and decode to utf-8 for JSON serialization
-    return {key: base64.b64encode(value).decode('utf-8') for key, value in params_dict.items()}
+    response = {str(param.get('key')): str(param.get('value')) for param in params_dict.get("params", [])}
+    response |= {"params": json.dumps(params_dict.get("params", []))} # Upcoming for settings v1
+    return response
 
   except Exception as e:
     cloudlog.exception("sunnylinkd.getParams.exception", e)

sunnypilot/sunnylink/backups/manager.py

@@ -20,6 +20,7 @@ from openpilot.system.version import get_version
 from cereal import messaging, custom
 from sunnypilot.sunnylink.api import SunnylinkApi
 from sunnypilot.sunnylink.backups.utils import decrypt_compressed_data, encrypt_compress_data, SnakeCaseEncoder
+from sunnypilot.sunnylink.utils import get_param_as_byte
 
 
 class OperationType(Enum):
@@ -74,7 +75,7 @@ class BackupManagerSP:
     config_data = {}
     params_to_backup = [k.decode('utf-8') for k in self.params.all_keys(ParamKeyFlag.BACKUP)]
     for param in params_to_backup:
-      value = str(self.params.get(param)).encode('utf-8')
+      value = get_param_as_byte(param)
       if value is not None:
         config_data[param] = base64.b64encode(value).decode('utf-8')
     return config_data
@@ -113,6 +114,7 @@ class BackupManagerSP:
       payload = json.loads(json.dumps(backup_info.to_dict(), cls=SnakeCaseEncoder))
       self._update_progress(75.0, OperationType.BACKUP)
 
+      cloudlog.debug(f"Uploading backup with payload: {json.dumps(payload)}")
       # Upload to sunnylink
       result = self.api.api_get(
         f"backup/{self.device_id}",
@@ -124,9 +126,11 @@ class BackupManagerSP:
       if result:
         self.backup_status = custom.BackupManagerSP.Status.completed
         self._update_progress(100.0, OperationType.BACKUP)
+        cloudlog.info("Backup successfully created and uploaded")
       else:
         self.backup_status = custom.BackupManagerSP.Status.failed
         self.last_error = "Failed to upload backup"
+        cloudlog.error(result)
         self._report_status()
 
       return bool(self.backup_status == custom.BackupManagerSP.Status.completed)
@@ -264,8 +268,8 @@ class BackupManagerSP:
         # Check for backup command
         if self.params.get_bool("BackupManager_CreateBackup"):
           try:
-            await self.create_backup()
-            reset_progress = True
+            if await self.create_backup():
+              reset_progress = True
           finally:
             self.params.remove("BackupManager_CreateBackup")
 

sunnypilot/sunnylink/utils.py

@@ -1,5 +1,6 @@
+import json
 from sunnypilot.sunnylink.api import SunnylinkApi, UNREGISTERED_SUNNYLINK_DONGLE_ID
-from openpilot.common.params import Params
+from openpilot.common.params import Params, ParamKeyType
 from openpilot.system.version import is_prebuilt
 
 
@@ -55,3 +56,15 @@ def get_api_token():
   sunnylink_api = SunnylinkApi(sunnylink_dongle_id)
   token = sunnylink_api.get_token()
   print(f"API Token: {token}")
+
+
+def get_param_as_byte(param_name: str) -> bytes:
+  params = Params()
+  param = params.get(param_name)
+  param_type = params.get_type(param_name)
+
+  if param_type == ParamKeyType.BYTES:
+    return bytes(param)
+  elif param_type == ParamKeyType.JSON:
+    return json.dumps(param).encode('utf-8')
+  return str(param).encode('utf-8')