Merge branch 'master' of github.com:commaai/agnos-builder into rmqtweston

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ repos:
     rev: v4.4.0
     hooks:
     -   id: check-ast
-        exclude: '^(tools)/|(userspace/usr/comma/reset)'
+        exclude: '^(tools)/|(userspace/usr/comma/reset)/|(userspace/usr/comma/updater)'
     -   id: check-toml
     -   id: check-xml
     -   id: check-yaml

TESTING.md

@@ -12,14 +12,11 @@
   - [ ] from openpilot menu
   - [ ] tapping on boot
   - [ ] corrupt userdata
-- [ ] Color calibration
-  - [ ] from /persist/comma/
-  - [ ] directly from panel over sysfs
+- [ ] Color calibration works from /persist/comma/
 - [ ] Clean setup: factory reset -> install openpilot -> openpilot works
 - [ ] AGNOS update works on warm boot
   - [ ] previous -> new
   - [ ] new -> previous
-- [ ] comma three: NVMe works
 - [ ] Display works at 60FPS
 
 ### ABL

VERSION

@@ -1 +1 @@
-13.1
+14.2

firmware/aop.img

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:21370172e590bd4ea907a558bcd6df20dc7a6c7d38b8e62fdde18f4a512ba9e9
+oid sha256:d8add1d4c1b6b443debf7bb80040e88a12140d248a328650d65ceaa0df04c1b7
 size 184364

firmware/devcfg.img

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d7d7e52963bbedbbf8a7e66847579ca106a0a729ce2cf60f4b8d8ea4b535d620
+oid sha256:7e8a836cf75a9097b1c78960d36f883699fcc3858d8a1d28338f889f6af25cc8
 size 40336

firmware/xbl.img

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:effa23294138e2297b85a5b482a885184c437b5ab25d74f2a62d4fce4e68f63b
+oid sha256:98e0642e2af77596e64d107b2c525a36e54d41d879268fd2fcab9255a2b29723
 size 3282256

firmware/xbl_config.img

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:63d019efed684601f145ef37628e62c8da73f5053a8e51d7de09e72b8b11f97c
+oid sha256:4d8add65e80b3e5ca49a64fac76025ee3a57a1523abd9caa407aa8c5fb721b0f
 size 98124

userspace/.gitattributes

@@ -5,3 +5,4 @@ files/libeglSubDriverWayland.so.patched filter=lfs diff=lfs merge=lfs -text
 usr/comma/reset filter=lfs diff=lfs merge=lfs -text
 usr/comma/setup filter=lfs diff=lfs merge=lfs -text
 usr/comma/installer filter=lfs diff=lfs merge=lfs -text
+usr/comma/updater filter=lfs diff=lfs merge=lfs -text

userspace/files/magic.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Magic
+
+[Service]
+Type=simple
+User=comma
+PermissionsStartOnly=true
+ExecStartPre=/bin/bash -c "chgrp gpu /dev/ion /dev/kgsl-3d0"
+ExecStartPre=/bin/bash -c "chmod 660 /dev/ion /dev/kgsl-3d0"
+
+ExecStart=/bin/bash -c "source /etc/profile && /usr/local/venv/bin/python -u /usr/comma/magic.py"
+
+[Install]
+WantedBy=multi-user.target

userspace/install_extras.sh

@@ -5,6 +5,7 @@
 apt-fast update && apt-fast install -y --no-install-recommends \
   bash-completion \
   btop \
+  hyperfine \
   iperf \
   iperf3 \
   dnsmasq \
@@ -13,6 +14,7 @@ apt-fast update && apt-fast install -y --no-install-recommends \
   ncdu \
   nfs-common \
   socat \
+  stress-ng \
   tree \
   wavemon \
   avahi-daemon \

userspace/usr/comma/comma.sh

@@ -8,6 +8,20 @@ CONTINUE="/data/continue.sh"
 INSTALLER="/tmp/installer"
 RESET_TRIGGER="/data/__system_reset__"
 
+echo "waiting for magic"
+for i in {1..200}; do
+  if systemctl is-active --quiet magic && [ -S /tmp/drmfd.sock ]; then
+    break
+  fi
+  sleep 0.1
+done
+
+if systemctl is-active --quiet magic && [ -S /tmp/drmfd.sock ]; then
+  echo "magic ready after ${SECONDS}s"
+else
+  echo "timed out waiting for magic, ${SECONDS}s"
+fi
+
 sudo chown comma: /data
 sudo chown comma: /data/media
 

userspace/usr/comma/installer

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:81fe860ae60a4058783b457b679d1a712bebbb32f8f3182f14a9a19235b6ef8a
-size 1263176
+oid sha256:ca13fd5e7ef06db10fefb613a101a1ea9cbb37794036e5f19600762210417725
+size 1295104

userspace/usr/comma/magic.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+import os, socket, struct, subprocess, threading, time
+from array import array
+
+import pyray as rl
+
+UPDATER_PATH = "/usr/comma/updater"
+WESTON_RUNTIME_DIR = "/var/tmp/weston"
+WESTON_SOCK_PATH = os.path.join(WESTON_RUNTIME_DIR, "wayland-0")
+
+SOCK_PATH = "/tmp/drmfd.sock"
+DRM_DEVICE = "/dev/dri/card0"
+BACKGROUND = "/usr/comma/bg.jpg"
+
+# This is needed to keep the old updater working. Updater used to be stored in
+# openpilot directly instead of in AGNOS. This will intercept the old updater
+# trying to use a Weston socket and start our own.
+def updater_weston():
+  os.makedirs(WESTON_RUNTIME_DIR, exist_ok=True)
+  os.chmod(WESTON_RUNTIME_DIR, 0o700)
+
+  try:
+    os.unlink(WESTON_SOCK_PATH)
+  except FileNotFoundError:
+    pass
+
+  server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+  server.bind(WESTON_SOCK_PATH)
+  server.listen(1)
+
+  while True:
+    try:
+      client, _ = server.accept()
+      creds = client.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, struct.calcsize("3i"))
+      pid, _, _ = struct.unpack("3i", creds)
+      with open(f"/proc/{pid}/comm", "r") as f:
+        comm = f.read().strip()
+        if comm == "updater":
+          with open(f"/proc/{pid}/cmdline", "rb") as f:
+            updater, manifest = [p.decode("utf-8") for p in f.read().split(b"\0") if p != b""][1:]
+            env = os.environ.copy()
+            env.pop("DRM_FD", None)
+            subprocess.run([UPDATER_PATH, updater, manifest], env=env)
+    except Exception:
+      pass
+    finally:
+      try:
+        client.shutdown(socket.SHUT_RDWR)
+        client.close()
+      except Exception:
+        pass
+
+def power_screen():
+  try:
+    with open("/sys/class/backlight/panel0-backlight/bl_power", "w") as f:
+      f.write("0")
+    with open("/sys/class/backlight/panel0-backlight/max_brightness") as f:
+      max_brightness = int(f.read().strip())
+    with open("/sys/class/backlight/panel0-backlight/brightness", "w") as f:
+      f.write(str(max_brightness))
+  except Exception:
+    pass
+
+def show_background(tex, pos):
+  rl.begin_drawing()
+  rl.draw_texture(tex, int(pos.x), int(pos.y), rl.WHITE)
+  rl.end_drawing()
+  power_screen()
+
+def handle_client(client, drm_master):
+  try:
+    drm_master_dup = os.dup(drm_master)
+    client.sendmsg([b"x"], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array("i", [drm_master_dup]).tobytes())])
+    client.recv(1)
+  except Exception:
+    pass
+  finally:
+    try:
+      os.close(drm_master_dup)
+      client.close()
+    except Exception:
+      pass
+
+def main():
+  threading.Thread(target=updater_weston, daemon=True).start()
+
+  while True:
+    try:
+      drm_master = os.open(DRM_DEVICE, os.O_RDWR | os.O_CLOEXEC)
+      break
+    except Exception as e:
+      print(e)
+      time.sleep(0.1)
+
+  os.environ['DRM_FD'] = str(drm_master)
+  rl.init_window(0, 0, "not weston")
+  img = rl.load_image(BACKGROUND)
+  rl.image_resize(img, rl.get_screen_width(), rl.get_screen_width()//2)
+  tex = rl.load_texture_from_image(img)
+  rl.set_texture_filter(tex, rl.TextureFilter.TEXTURE_FILTER_BILINEAR)
+  pos = rl.Vector2((rl.get_screen_width() - tex.width)/2.0, (rl.get_screen_height() - tex.height)/2.0)
+  rl.unload_image(img)
+  show_background(tex, pos)
+
+  try:
+    os.unlink(SOCK_PATH)
+  except FileNotFoundError:
+    pass
+
+  server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+  server.bind(SOCK_PATH)
+  server.settimeout(0.1)
+  server.listen(1)
+
+  clients = set()
+  need_background = False
+
+  while True:
+    dead = [t for t in list(clients) if not t.is_alive()]
+    for t in dead:
+      t.join()
+      clients.discard(t)
+    if not clients and need_background:
+      need_background = False
+      show_background(tex, pos)
+
+    try:
+      client, _ = server.accept()
+    except Exception:
+      continue
+
+    need_background = True
+    t = threading.Thread(target=handle_client, args=(client, drm_master), daemon=True)
+    t.start()
+    clients.add(t)
+
+if __name__ == "__main__":
+  main()

userspace/usr/comma/power_monitor.py

@@ -4,10 +4,12 @@ import time
 import fcntl
 import struct
 import threading
+import subprocess
 from datetime import timedelta
 
 FN = "/var/tmp/power_watchdog"
 THRESHOLD = timedelta(hours=1.0)
+timestamps = {}
 
 
 def read(path: str, num: bool = False):
@@ -20,9 +22,8 @@ def read(path: str, num: bool = False):
     return 0 if num else ""
 
 
-last_touch_ts = 0
 def check_touches():
-  global last_touch_ts
+  global timestamps
 
   event_format = "llHHi"
   event_size = struct.calcsize(event_format)
@@ -33,9 +34,12 @@ def check_touches():
       while (event := event_file.read(event_size)):
         (sec, usec, etype, code, value) = struct.unpack(event_format, event)
         if etype != 0 or code != 0 or value != 0:
-          last_touch_ts = time.monotonic()
+          timestamps['touch'] = time.monotonic()
       time.sleep(60)
 
+def ssh_active():
+  p = subprocess.run("ss | grep ssh", shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+  return p.returncode == 0
 
 if __name__ == "__main__":
   # we limit worst-case power usage when openpilot isn't managing it,
@@ -43,17 +47,18 @@ if __name__ == "__main__":
 
   threading.Thread(target=check_touches, daemon=True).start()
 
-  last_valid_readout = time.monotonic()
+  timestamps['startup'] = time.monotonic()
   while True:
-    cur_t = read(FN, True)
-    last_valid_readout = max(last_valid_readout, last_touch_ts, cur_t)
-
-    not_engaged = not read("/data/params/d/IsEngaged").startswith("1")
+    timestamps['watchdog'] = read(FN, True)
+    if ssh_active():
+      timestamps['ssh'] = time.monotonic()
+    if read("/data/params/d/IsEngaged").startswith("1"):
+      timestamps['engaged'] = time.monotonic()
 
     # time to shutoff?
-    dt = timedelta(seconds=time.monotonic() - last_valid_readout)
-    if dt > THRESHOLD and not_engaged:
+    dt = timedelta(seconds=time.monotonic() - max(timestamps.values()))
+    if dt > THRESHOLD:
       os.system("sudo poweroff")
 
-    print((THRESHOLD - dt), "until shutdown", "/ not engaged:", not_engaged)
+    print((THRESHOLD - dt), "until shutdown", f"/ {timestamps=}")
     time.sleep(60)

userspace/usr/comma/reset

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c8461a4a429f8ebc93d26061a15f1b609b84d3645663dd4055f46257815dc771
-size 20579091
+oid sha256:d8afce33a9fce602024260f485bbcd63d5a396648aa643a5043bfa00140ac31b
+size 20780740

userspace/usr/comma/setup

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:37ede2d4dee0e041d679efc1e7ddc1965d9af56200398b3f82c938bff3aea55f
-size 20581042
+oid sha256:91c253b2217a1bdd340cbebec0f6b456315c2357d762e489d809fb11d200bef4
+size 20785145

userspace/uv/openpilot/pyproject.toml

@@ -23,7 +23,7 @@ dependencies = [
   # core
   "cffi",
   "scons",
-  "pycapnp",
+  "pycapnp==2.1.0",
   "Cython",
   "setuptools",
   "numpy >=2.0",
@@ -72,7 +72,9 @@ dependencies = [
   "zstandard",
 
   # ui
+  "raylib < 5.5.0.3", # TODO: unpin when they fix https://github.com/electronstudio/raylib-python-cffi/issues/186
   "qrcode",
+  "mapbox-earcut",
 ]
 
 [project.optional-dependencies]
@@ -119,11 +121,11 @@ dev = [
   "tabulate",
   "types-requests",
   "types-tabulate",
-  "raylib",
 ]
 
 tools = [
   "metadrive-simulator @ https://github.com/commaai/metadrive/releases/download/MetaDrive-minimal-0.4.2.4/metadrive_simulator-0.4.2.4-py3-none-any.whl ; (platform_machine != 'aarch64')",
+  "dearpygui>=2.1.0",
 ]
 
 [project.urls]
@@ -175,7 +177,7 @@ quiet-level = 3
 # if you've got a short variable name that's getting flagged, add it here
 ignore-words-list = "bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie,preints,whit,indexIn,ws,uint,grey,deque,stdio,amin,BA,LITE,atEnd,UIs,errorString,arange,FocusIn,od,tim,relA,hist,copyable,jupyter,thead,TGE,abl,lite"
 builtin = "clear,rare,informal,code,names,en-GB_to_en-US"
-skip = "./third_party/*, ./tinygrad/*, ./tinygrad_repo/*, ./msgq/*, ./panda/*, ./opendbc/*, ./opendbc_repo/*, ./rednose/*, ./rednose_repo/*, ./teleoprtc/*, ./teleoprtc_repo/*, *.ts, uv.lock, *.onnx, ./cereal/gen/*, */c_generated_code/*, docs/assets/*"
+skip = "./third_party/*, ./tinygrad/*, ./tinygrad_repo/*, ./msgq/*, ./panda/*, ./opendbc/*, ./opendbc_repo/*, ./rednose/*, ./rednose_repo/*, ./teleoprtc/*, ./teleoprtc_repo/*, *.ts, uv.lock, *.onnx, ./cereal/gen/*, */c_generated_code/*, docs/assets/*, tools/plotjuggler/layouts/*"
 
 [tool.mypy]
 python_version = "3.11"
@@ -261,8 +263,13 @@ lint.flake8-implicit-str-concat.allow-multiline = false
 "tools".msg = "Use openpilot.tools"
 "pytest.main".msg = "pytest.main requires special handling that is easy to mess up!"
 "unittest".msg = "Use pytest"
-"pyray.measure_text_ex".msg = "Use openpilot.system.ui.lib.text_measure"
 "time.time".msg = "Use time.monotonic"
 
+# raylib banned APIs
+"pyray.measure_text_ex".msg = "Use openpilot.system.ui.lib.text_measure"
+"pyray.is_mouse_button_pressed".msg = "This can miss events. Use Widget._handle_mouse_press"
+"pyray.is_mouse_button_released".msg = "This can miss events. Use Widget._handle_mouse_release"
+"pyray.draw_text".msg = "Use a function (such as rl.draw_font_ex) that takes font as an argument"
+
 [tool.ruff.format]
 quote-style = "preserve"

userspace/uv/pyproject.toml

@@ -32,7 +32,7 @@ dependencies = [
   "flaky",
 
   # GUIs
-  "raylib @ https://github.com/commaai/raylib-python-cffi/releases/download/3/raylib-5.5.0.2-cp312-cp312-linux_aarch64.whl",
+  "raylib @ https://github.com/commaai/raylib-python-cffi/releases/download/5/raylib-5.5.0.2-cp312-cp312-linux_aarch64.whl",
 ]
 
 [tool.uv.sources]