params: auto decode based on type (#35794)

* type

* test

* more

* might as well use this

* one more

* live

* athena

* b

* also

* more

* now

* ah

* pigeon
This commit is contained in:
Maxime Desroches
2025-07-22 21:58:06 -07:00
committed by GitHub
parent dc1219d13f
commit bc5336d805
31 changed files with 93 additions and 92 deletions

View File

@@ -21,12 +21,13 @@ enum ParamKeyFlag {
};
enum ParamKeyType {
STRING = 0,
STRING = 0, // must be utf-8 decodable
BOOL = 1,
INT = 2,
FLOAT = 3,
TIME = 4, // ISO 8601
JSON = 5
JSON = 5,
BYTES = 6
};
struct ParamKeyAttributes {

View File

@@ -16,14 +16,14 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
{"AthenadUploadQueue", {PERSISTENT, JSON}},
{"AthenadRecentlyViewedRoutes", {PERSISTENT, STRING}},
{"BootCount", {PERSISTENT, INT}},
{"CalibrationParams", {PERSISTENT, STRING}},
{"CalibrationParams", {PERSISTENT, BYTES}},
{"CameraDebugExpGain", {CLEAR_ON_MANAGER_START, STRING}},
{"CameraDebugExpTime", {CLEAR_ON_MANAGER_START, STRING}},
{"CarBatteryCapacity", {PERSISTENT, INT}},
{"CarParams", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, STRING}},
{"CarParamsCache", {CLEAR_ON_MANAGER_START, STRING}},
{"CarParamsPersistent", {PERSISTENT, STRING}},
{"CarParamsPrevRoute", {PERSISTENT, STRING}},
{"CarParams", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BYTES}},
{"CarParamsCache", {CLEAR_ON_MANAGER_START, BYTES}},
{"CarParamsPersistent", {PERSISTENT, BYTES}},
{"CarParamsPrevRoute", {PERSISTENT, BYTES}},
{"CompletedTrainingVersion", {PERSISTENT, STRING, "0"}},
{"ControlsReady", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, BOOL}},
{"CurrentBootlog", {PERSISTENT, STRING}},
@@ -74,11 +74,11 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
{"LastPowerDropDetected", {CLEAR_ON_MANAGER_START, STRING}},
{"LastUpdateException", {CLEAR_ON_MANAGER_START, STRING}},
{"LastUpdateTime", {PERSISTENT, TIME}},
{"LiveDelay", {PERSISTENT, STRING}},
{"LiveParameters", {PERSISTENT, STRING}},
{"LiveParametersV2", {PERSISTENT, STRING}},
{"LiveTorqueParameters", {PERSISTENT | DONT_LOG, STRING}},
{"LocationFilterInitialState", {PERSISTENT, STRING}},
{"LiveDelay", {PERSISTENT, BYTES}},
{"LiveParameters", {PERSISTENT, BYTES}},
{"LiveParametersV2", {PERSISTENT, BYTES}},
{"LiveTorqueParameters", {PERSISTENT | DONT_LOG, BYTES}},
{"LocationFilterInitialState", {PERSISTENT, BYTES}},
{"LongitudinalManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
{"LongitudinalPersonality", {PERSISTENT, INT, std::to_string(static_cast<int>(cereal::LongitudinalPersonality::STANDARD))}},
{"NetworkMetered", {PERSISTENT, BOOL}},
@@ -99,7 +99,7 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
{"OpenpilotEnabledToggle", {PERSISTENT, BOOL, "1"}},
{"PandaHeartbeatLost", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
{"PandaSomResetTriggered", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
{"PandaSignatures", {CLEAR_ON_MANAGER_START, STRING}},
{"PandaSignatures", {CLEAR_ON_MANAGER_START, BYTES}},
{"PrimeType", {PERSISTENT, INT}},
{"RecordAudio", {PERSISTENT, BOOL}},
{"RecordFront", {PERSISTENT, BOOL}},

View File

@@ -23,6 +23,7 @@ cdef extern from "common/params.h":
FLOAT
TIME
JSON
BYTES
cdef cppclass c_Params "Params":
c_Params(string) except + nogil
@@ -72,26 +73,7 @@ cdef class Params:
raise UnknownKeyName(key)
return key
def cast(self, value, t, default):
try:
if t == STRING:
return value
elif t == BOOL:
return value == b"1"
elif t == INT:
return int(value)
elif t == FLOAT:
return float(value)
elif t == TIME:
return datetime.datetime.fromisoformat(value)
elif t == JSON:
return json.loads(value)
else:
return default
except (TypeError, ValueError):
return default
def get(self, key, bool block=False, encoding=None, default=None):
def get(self, key, bool block=False, default=None):
cdef string k = self.check_key(key)
cdef ParamKeyType t = self.p.getKeyType(ensure_bytes(key))
cdef string val
@@ -106,7 +88,25 @@ cdef class Params:
else:
return default
return self.cast(val if encoding is None else val.decode(encoding), t, default)
try:
if t == STRING:
return val.decode("utf-8")
elif t == BOOL:
return val == b"1"
elif t == INT:
return int(val)
elif t == FLOAT:
return float(val)
elif t == TIME:
return datetime.datetime.fromisoformat(val.decode("utf-8"))
elif t == JSON:
return json.loads(val)
elif t == BYTES:
return val
else:
return default
except (TypeError, ValueError):
return default
def get_bool(self, key, bool block=False):
cdef string k = self.check_key(key)

View File

@@ -14,7 +14,7 @@ class TestParams:
def test_params_put_and_get(self):
self.params.put("DongleId", "cb38263377b873ee")
assert self.params.get("DongleId") == b"cb38263377b873ee"
assert self.params.get("DongleId") == "cb38263377b873ee"
def test_params_non_ascii(self):
st = b"\xe1\x90\xff"
@@ -39,8 +39,8 @@ class TestParams:
def test_params_two_things(self):
self.params.put("DongleId", "bob")
self.params.put("AthenadPid", "123")
assert self.params.get("DongleId") == b"bob"
assert self.params.get("AthenadPid") == b"123"
assert self.params.get("DongleId") == "bob"
assert self.params.get("AthenadPid") == "123"
def test_params_get_block(self):
def _delayed_writer():
@@ -131,14 +131,14 @@ class TestParams:
# time
now = datetime.datetime.now(datetime.UTC)
self.params.put("InstallDate", str(now))
assert self.params.get("InstallDate", encoding="utf-8") == now
assert self.params.get("InstallDate") == now
def test_params_get_default(self):
now = datetime.datetime.now(datetime.UTC)
self.params.remove("InstallDate")
assert self.params.get("InstallDate", encoding="utf-8") is None
assert self.params.get("InstallDate", encoding="utf-8", default=now) == now
assert self.params.get("InstallDate") is None
assert self.params.get("InstallDate", default=now) == now
self.params.put("BootCount", "1xx1")
assert self.params.get("BootCount", encoding="utf-8") is None
assert self.params.get("BootCount", encoding="utf-8", default=1441) == 1441
assert self.params.get("BootCount") is None
assert self.params.get("BootCount", default=1441) == 1441

View File

@@ -128,7 +128,7 @@ class Car:
except Exception:
pass
secoc_key = self.params.get("SecOCKey", encoding='utf8')
secoc_key = self.params.get("SecOCKey")
if secoc_key is not None:
saved_secoc_key = bytes.fromhex(secoc_key.strip())
if len(saved_secoc_key) == 16:

View File

@@ -111,7 +111,7 @@ class TestAlerts:
alert = copy.copy(self.offroad_alerts[a])
set_offroad_alert(a, True)
alert['extra'] = ''
assert alert == params.get(a, encoding='utf8')
assert alert == params.get(a)
# then delete it
set_offroad_alert(a, False)
@@ -125,6 +125,6 @@ class TestAlerts:
alert = self.offroad_alerts[a]
set_offroad_alert(a, True, extra_text="a"*i)
written_alert = params.get(a, encoding='utf8')
written_alert = params.get(a)
assert "a"*i == written_alert['extra']
assert alert["text"] == written_alert['text']

View File

@@ -147,7 +147,7 @@ class TestOnroad:
while not sm.seen['carState']:
sm.update(1000)
route = params.get("CurrentRoute", encoding="utf-8")
route = params.get("CurrentRoute")
assert route is not None
segs = list(Path(Paths.log_root()).glob(f"{route}--*"))

View File

@@ -105,7 +105,7 @@ class TestUpdated:
ret = None
start_time = time.monotonic()
while ret is None:
ret = self.params.get(key, encoding='utf8')
ret = self.params.get(key)
if time.monotonic() - start_time > timeout:
break
time.sleep(0.01)
@@ -162,7 +162,7 @@ class TestUpdated:
def _check_update_state(self, update_available):
# make sure LastUpdateTime is recent
last_update_time = self._read_param("LastUpdateTime", encoding="utf-8")
last_update_time = self._read_param("LastUpdateTime")
td = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) - last_update_time
assert td.total_seconds() < 10
self.params.remove("LastUpdateTime")

View File

@@ -210,5 +210,5 @@ class HomeLayout(Widget):
def _get_version_text(self) -> str:
brand = "openpilot"
description = self.params.get("UpdaterCurrentDescription", encoding='utf-8')
description = self.params.get("UpdaterCurrentDescription")
return f"{brand} {description}" if description else brand

View File

@@ -41,8 +41,8 @@ class DeviceLayout(Widget):
self._scroller = Scroller(items, line_separator=True, spacing=0)
def _initialize_items(self):
dongle_id = self._params.get("DongleId", encoding="utf-8") or "N/A"
serial = self._params.get("HardwareSerial") or "N/A"
dongle_id = self._params.get("DongleId", default="N/A")
serial = self._params.get("HardwareSerial", default="N/A")
items = [
text_item("Dongle ID", dongle_id),

View File

@@ -51,7 +51,7 @@ class FirehoseLayout(Widget):
self.last_update_time = 0
def _get_segment_count(self) -> int:
stats = self.params.get(self.PARAM_KEY, encoding='utf8')
stats = self.params.get(self.PARAM_KEY)
if not stats:
return 0
try:
@@ -161,7 +161,7 @@ class FirehoseLayout(Widget):
def _fetch_firehose_stats(self):
try:
dongle_id = self.params.get("DongleId", encoding='utf8')
dongle_id = self.params.get("DongleId")
if not dongle_id or dongle_id == UNREGISTERED_DONGLE_ID:
return
identity_token = Api(dongle_id).get_token()

View File

@@ -54,7 +54,7 @@ class TogglesLayout(Widget):
buttons=["Aggressive", "Standard", "Relaxed"],
button_width=255,
callback=self._set_longitudinal_personality,
selected_index=int(self._params.get("LongitudinalPersonality") or 0),
selected_index=self._params.get("LongitudinalPersonality", default=0),
icon="speed_limit.png"
),
toggle_item(

View File

@@ -35,7 +35,7 @@ class PrimeState:
self.start()
def _load_initial_state(self) -> PrimeType:
prime_type_str = os.getenv("PRIME_TYPE") or self._params.get("PrimeType", encoding='utf8')
prime_type_str = os.getenv("PRIME_TYPE") or self._params.get("PrimeType")
try:
if prime_type_str is not None:
return PrimeType(int(prime_type_str))
@@ -44,7 +44,7 @@ class PrimeState:
return PrimeType.UNKNOWN
def _fetch_prime_status(self) -> None:
dongle_id = self._params.get("DongleId", encoding='utf8')
dongle_id = self._params.get("DongleId")
if not dongle_id or dongle_id == UNREGISTERED_DONGLE_ID:
return

View File

@@ -292,7 +292,7 @@ class UpdateAlert(AbstractAlert):
def refresh(self) -> bool:
update_available: bool = self.params.get_bool("UpdateAvailable")
if update_available:
self.release_notes = self.params.get("UpdaterNewReleaseNotes", encoding='utf-8')
self.release_notes = self.params.get("UpdaterNewReleaseNotes")
self._cached_content_height = 0
return update_available

View File

@@ -23,7 +23,7 @@ class PairingDialog:
def _get_pairing_url(self) -> str:
try:
dongle_id = self.params.get("DongleId", encoding='utf8') or ""
dongle_id = self.params.get("DongleId", default="")
token = Api(dongle_id).get_token()
except Exception as e:
cloudlog.warning(f"Failed to get pairing token: {e}")

View File

@@ -470,7 +470,7 @@ def setRouteViewed(route: str) -> dict[str, int | str]:
# maintain a list of the last 10 routes viewed in connect
params = Params()
r = params.get("AthenadRecentlyViewedRoutes", encoding="utf8")
r = params.get("AthenadRecentlyViewedRoutes")
routes = [] if r is None else r.split(",")
routes.append(route)
@@ -492,7 +492,7 @@ def startLocalProxy(global_end_event: threading.Event, remote_ws_uri: str, local
cloudlog.debug("athena.startLocalProxy.starting")
dongle_id = Params().get("DongleId").decode('utf8')
dongle_id = Params().get("DongleId")
identity_token = Api(dongle_id).get_token()
ws = create_connection(remote_ws_uri,
cookie="jwt=" + identity_token,
@@ -532,12 +532,12 @@ def getPublicKey() -> str | None:
@dispatcher.add_method
def getSshAuthorizedKeys() -> str:
return Params().get("GithubSshKeys", encoding='utf8') or ''
return cast(str, Params().get("GithubSshKeys", default=""))
@dispatcher.add_method
def getGithubUsername() -> str:
return Params().get("GithubUsername", encoding='utf8') or ''
return cast(str, Params().get("GithubUsername", default=""))
@dispatcher.add_method
def getSimInfo():
@@ -815,7 +815,7 @@ def main(exit_event: threading.Event = None):
cloudlog.exception("failed to set core affinity")
params = Params()
dongle_id = params.get("DongleId", encoding='utf-8')
dongle_id = params.get("DongleId")
UploadQueueCache.initialize(upload_queue)
ws_uri = ATHENA_HOST + "/ws/v2/" + dongle_id

View File

@@ -14,7 +14,7 @@ ATHENA_MGR_PID_PARAM = "AthenadPid"
def main():
params = Params()
dongle_id = params.get("DongleId").decode('utf-8')
dongle_id = params.get("DongleId")
build_metadata = get_build_metadata()
cloudlog.bind_global(dongle_id=dongle_id,

View File

@@ -17,7 +17,7 @@ from openpilot.common.swaglog import cloudlog
UNREGISTERED_DONGLE_ID = "UnregisteredDevice"
def is_registered_device() -> bool:
dongle = Params().get("DongleId", encoding='utf-8')
dongle = Params().get("DongleId")
return dongle not in (None, UNREGISTERED_DONGLE_ID)
@@ -33,7 +33,7 @@ def register(show_spinner=False) -> str | None:
"""
params = Params()
dongle_id: str | None = params.get("DongleId", encoding='utf8')
dongle_id: str | None = params.get("DongleId")
if dongle_id is None and Path(Paths.persist_root()+"/comma/dongle_id").is_file():
# not all devices will have this; added early in comma 3X production (2/28/24)
with open(Paths.persist_root()+"/comma/dongle_id") as f:

View File

@@ -28,7 +28,7 @@ class TestAthenadPing:
exit_event: threading.Event
def _get_ping_time(self) -> str | None:
return cast(str | None, self.params.get("LastAthenaPingTime", encoding="utf-8"))
return cast(str | None, self.params.get("LastAthenaPingTime"))
def _clear_ping_time(self) -> None:
self.params.remove("LastAthenaPingTime")
@@ -42,7 +42,7 @@ class TestAthenadPing:
def setup_method(self) -> None:
self.params = Params()
self.dongle_id = self.params.get("DongleId", encoding="utf-8")
self.dongle_id = self.params.get("DongleId")
wifi_radio(True)
self._clear_ping_time()

View File

@@ -49,7 +49,7 @@ class TestRegistration:
dongle = register()
assert m.call_count == 0
assert dongle == UNREGISTERED_DONGLE_ID
assert self.params.get("DongleId", encoding='utf-8') == dongle
assert self.params.get("DongleId") == dongle
def test_missing_cache(self, mocker):
# keys exist but no dongle id
@@ -63,7 +63,7 @@ class TestRegistration:
# call again, shouldn't hit the API this time
assert register() == dongle
assert m.call_count == 1
assert self.params.get("DongleId", encoding='utf-8') == dongle
assert self.params.get("DongleId") == dongle
def test_unregistered(self, mocker):
# keys exist, but unregistered
@@ -73,4 +73,4 @@ class TestRegistration:
dongle = register()
assert m.call_count == 1
assert dongle == UNREGISTERED_DONGLE_ID
assert self.params.get("DongleId", encoding='utf-8') == dongle
assert self.params.get("DongleId") == dongle

View File

@@ -400,7 +400,7 @@ def hardware_thread(end_event, hw_queue) -> None:
last_ping = params.get("LastAthenaPingTime")
if last_ping is not None:
msg.deviceState.lastAthenaPingTime = int(last_ping)
msg.deviceState.lastAthenaPingTime = last_ping
msg.deviceState.thermalStatus = thermal_status
pm.send("deviceState", msg)

View File

@@ -88,7 +88,7 @@ class Uploader:
self.immediate_priority = {"qlog": 0, "qlog.zst": 0, "qcamera.ts": 1}
def list_upload_files(self, metered: bool) -> Iterator[tuple[str, str, str]]:
r = self.params.get("AthenadRecentlyViewedRoutes", encoding="utf8")
r = self.params.get("AthenadRecentlyViewedRoutes")
requested_routes = [] if r is None else [route for route in r.split(",") if route]
for logdir in listdir_by_creation(self.root):
@@ -238,7 +238,7 @@ def main(exit_event: threading.Event = None) -> None:
clear_locks(Paths.log_root())
params = Params()
dongle_id = params.get("DongleId", encoding='utf8')
dongle_id = params.get("DongleId")
if dongle_id is None:
cloudlog.info("uploader missing dongle_id")

View File

@@ -112,7 +112,7 @@ def manager_thread() -> None:
params = Params()
ignore: list[str] = []
if params.get("DongleId", encoding='utf8') in (None, UNREGISTERED_DONGLE_ID):
if params.get("DongleId") in (None, UNREGISTERED_DONGLE_ID):
ignore += ["manage_athenad", "uploader"]
if os.getenv("NOBOARD") is not None:
ignore.append("pandad")

View File

@@ -251,7 +251,7 @@ class DaemonProcess(ManagerProcess):
if self.params is None:
self.params = Params()
pid = self.params.get(self.param_name, encoding='utf-8')
pid = self.params.get(self.param_name)
if pid is not None:
try:
os.kill(int(pid), 0)

View File

@@ -49,7 +49,7 @@ def init(project: SentryProject) -> bool:
return False
env = "release" if build_metadata.tested_channel else "master"
dongle_id = Params().get("DongleId", encoding='utf-8')
dongle_id = Params().get("DongleId")
integrations = []
if project == SentryProject.SELFDRIVE:

View File

@@ -61,7 +61,7 @@ class StatLog:
def main() -> NoReturn:
dongle_id = Params().get("DongleId", encoding='utf-8')
dongle_id = Params().get("DongleId")
def get_influxdb_line(measurement: str, value: float | dict[str, float], timestamp: datetime, tags: dict) -> str:
res = f"{measurement}"
for k, v in tags.items():

View File

@@ -41,7 +41,7 @@ def add_ubx_checksum(msg: bytes) -> bytes:
B = (B + A) % 256
return msg + bytes([A, B])
def get_assistnow_messages(token: bytes) -> list[bytes]:
def get_assistnow_messages(token: str) -> list[bytes]:
# make request
# TODO: implement adding the last known location
r = requests.get("https://online-live2.services.u-blox.com/GetOnlineData.ashx", params=urllib.parse.urlencode({

View File

@@ -91,7 +91,7 @@ class WifiManager:
# Set tethering ssid as "weedle" + first 4 characters of a dongle id
self._tethering_ssid = "weedle"
if Params is not None:
dongle_id = Params().get("DongleId", encoding="utf-8")
dongle_id = Params().get("DongleId")
if dongle_id:
self._tethering_ssid += "-" + dongle_id[:4]
self.running: bool = True

View File

@@ -132,8 +132,8 @@ class TestBaseUpdate:
class ParamsBaseUpdateTest(TestBaseUpdate):
def _test_finalized_update(self, branch, version, agnos_version, release_notes):
assert self.params.get("UpdaterNewDescription", encoding="utf-8").startswith(f"{version} / {branch}")
assert self.params.get("UpdaterNewReleaseNotes", encoding="utf-8") == f"{release_notes}\n"
assert self.params.get("UpdaterNewDescription").startswith(f"{version} / {branch}")
assert self.params.get("UpdaterNewReleaseNotes") == f"{release_notes}\n"
super()._test_finalized_update(branch, version, agnos_version, release_notes)
def send_check_for_updates_signal(self, updated: ManagerProcess):
@@ -143,16 +143,16 @@ class ParamsBaseUpdateTest(TestBaseUpdate):
updated.signal(signal.SIGHUP.value)
def _test_params(self, branch, fetch_available, update_available):
assert self.params.get("UpdaterTargetBranch", encoding="utf-8") == branch
assert self.params.get("UpdaterTargetBranch") == branch
assert self.params.get_bool("UpdaterFetchAvailable") == fetch_available
assert self.params.get_bool("UpdateAvailable") == update_available
def wait_for_idle(self):
self.wait_for_condition(lambda: self.params.get("UpdaterState", encoding="utf-8") == "idle")
self.wait_for_condition(lambda: self.params.get("UpdaterState") == "idle")
def wait_for_failed(self):
self.wait_for_condition(lambda: self.params.get("UpdateFailedCount", encoding="utf-8") is not None and \
self.params.get("UpdateFailedCount", encoding="utf-8") > 0)
self.wait_for_condition(lambda: self.params.get("UpdateFailedCount") is not None and \
self.params.get("UpdateFailedCount") > 0)
def wait_for_fetch_available(self):
self.wait_for_condition(lambda: self.params.get_bool("UpdaterFetchAvailable"))

View File

@@ -234,7 +234,7 @@ class Updater:
@property
def target_branch(self) -> str:
b: str | None = self.params.get("UpdaterTargetBranch", encoding='utf-8')
b: str | None = self.params.get("UpdaterTargetBranch")
if b is None:
b = self.get_branch(BASEDIR)
return b
@@ -275,7 +275,7 @@ class Updater:
if update_success:
write_time_to_param(self.params, "LastUpdateTime")
else:
t = self.params.get("LastUpdateTime", encoding="utf8")
t = self.params.get("LastUpdateTime")
if t is not None:
last_update = t
@@ -420,7 +420,7 @@ def main() -> None:
if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir():
cloudlog.event("update installed")
if not params.get("InstallDate", encoding="utf-8"):
if not params.get("InstallDate"):
t = datetime.datetime.now(datetime.UTC).replace(tzinfo=None).isoformat()
params.put("InstallDate", t.encode('utf8'))
@@ -460,7 +460,7 @@ def main() -> None:
updater.check_for_update()
# download update
last_fetch = params.get("UpdaterLastFetchTime", encoding="utf8")
last_fetch = params.get("UpdaterLastFetchTime")
timed_out = last_fetch is None or (datetime.datetime.now(datetime.UTC).replace(tzinfo=None) - last_fetch > datetime.timedelta(days=3))
user_requested_fetch = wait_helper.user_request == UserRequest.FETCH
if params.get_bool("NetworkMetered") and not timed_out and not user_requested_fetch:

View File

@@ -15,8 +15,8 @@ TESTED_BRANCHES = RELEASE_BRANCHES + ['devel', 'devel-staging', 'nightly-dev']
BUILD_METADATA_FILENAME = "build.json"
training_version: bytes = b"0.2.0"
terms_version: bytes = b"2"
training_version: str = "0.2.0"
terms_version: str = "2"
def get_version(path: str = BASEDIR) -> str: