Update to Python 3.12 (#32548)

* 3.12

* pprofile is broken

* use modified metadrivepy3-12

* 0.3.0 metadrive

* add metadrive/commaai git dependency

* metadrive git set

* pin sounddevice 0.4.6

* datetime.utcnow() deprecation

* poetry lock

* make datetime not aware

* poetry lock

* pin pytools

* google_crc32c wheel

* unpin sounddevice

* clean metadrive

* use python crc

* mypy

* 3.12.4

* allow python3.11

* test

* no pip

* poetry

* better

* better

* merge

* remove

* try

* test

* try  this

* snok

* python

* simpler

* setuptools

* lower

* try

* try

* work?

* ubuntu deps

* ubuntu

* try

* remove

* move

* remove this

* names

* Update .github/workflows/tools_tests.yaml

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>

* python<4

* <3.13

---------

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>
Co-authored-by: Maxime Desroches <desroches.maxime@gmail.com>

.github/workflows/tools_tests.yaml

@@ -39,6 +39,25 @@ jobs:
                         source selfdrive/test/setup_vsound.sh && \
                         CI=1 pytest tools/sim/tests/test_metadrive_bridge.py"
 
+  test_python311:
+    name: test python3.11 support
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+    - uses: actions/checkout@v4
+    - name: Installing ubuntu dependencies
+      run: INSTALL_EXTRA_PACKAGES=no tools/install_ubuntu_dependencies.sh
+    - name: Installing python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.11.4'
+    - name: Installing pip
+      run: pip install pip==24.0
+    - name: Installing poetry
+      run: pip install poetry==1.7.0
+    - name: Installing python dependencies
+      run: poetry install --no-cache --no-root
+
   devcontainer:
     name: devcontainer
     runs-on: ubuntu-latest

.python-version

@@ -1 +1 @@
-3.11.4
+3.12.4

Dockerfile.openpilot_base

@@ -64,7 +64,7 @@ RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 USER $USER
 
 ENV POETRY_VIRTUALENVS_CREATE=false
-ENV PYENV_VERSION=3.11.4
+ENV PYENV_VERSION=3.12.4
 ENV PYENV_ROOT="/home/$USER/pyenv"
 ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/shims:$PATH"
 
@@ -76,7 +76,7 @@ RUN cd /tmp && \
     rm -rf /tmp/* && \
     rm -rf /home/$USER/.cache && \
     find /home/$USER/pyenv -type d -name ".git" | xargs rm -rf && \
-    rm -rf /home/$USER/pyenv/versions/3.11.4/lib/python3.11/test
+    rm -rf /home/$USER/pyenv/versions/3.12.4/lib/python3.12/test
 
 USER root
 RUN sudo git config --global --add safe.directory /tmp/openpilot

common/api/__init__.py

@@ -1,7 +1,7 @@
 import jwt
 import os
 import requests
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, UTC
 from openpilot.system.hardware.hw import Paths
 from openpilot.system.version import get_version
 
@@ -23,7 +23,7 @@ class Api:
     return api_get(endpoint, method=method, timeout=timeout, access_token=access_token, **params)
 
   def get_token(self, expiry_hours=1):
-    now = datetime.utcnow()
+    now = datetime.now(UTC).replace(tzinfo=None)
     payload = {
       'identity': self.dongle_id,
       'nbf': now,

pyproject.toml

@@ -88,7 +88,7 @@ repository = "https://github.com/commaai/openpilot"
 documentation = "https://docs.comma.ai"
 
 [tool.poetry.dependencies]
-python = "~3.11"
+python = ">=3.11, <3.13"
 
 # multiple users
 sounddevice = "*"  # micd + soundd
@@ -154,15 +154,14 @@ inputs = "*"
 Jinja2 = "*"
 lru-dict = "*"
 matplotlib = "*"
-# No release for this fix https://github.com/metadriverse/metadrive/issues/632. Pinned to this commit until next release
-metadrive-simulator = {git = "https://github.com/metadriverse/metadrive.git", rev ="233a3a1698be7038ec3dd050ca10b547b4b3324c", markers = "platform_machine != 'aarch64'" } # no linux/aarch64 wheels for certain dependencies
+metadrive-simulator = { git = "https://github.com/commaai/metadrive.git", branch = "python3.12", markers = "platform_machine != 'aarch64'" } # no linux/aarch64 wheels for certain dependencies
 mpld3 = "*"
 mypy = "*"
 myst-parser = "*"
 natsort = "*"
 opencv-python-headless = "*"
 parameterized = "^0.8"
-pprofile = "*"
+#pprofile = "*"
 polyline = "*"
 pre-commit = "*"
 pyautogui = "*"

selfdrive/test/test_updated.py

@@ -164,7 +164,7 @@ class TestUpdated:
     # make sure LastUpdateTime is recent
     t = self._read_param("LastUpdateTime")
     last_update_time = datetime.datetime.fromisoformat(t)
-    td = datetime.datetime.utcnow() - last_update_time
+    td = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) - last_update_time
     assert td.total_seconds() < 10
     self.params.remove("LastUpdateTime")
 

system/athena/registration.py

@@ -4,7 +4,7 @@ import json
 import jwt
 from pathlib import Path
 
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, UTC
 from openpilot.common.api import api_get
 from openpilot.common.params import Params
 from openpilot.common.spinner import Spinner
@@ -66,7 +66,7 @@ def register(show_spinner=False) -> str | None:
     start_time = time.monotonic()
     while True:
       try:
-        register_token = jwt.encode({'register': True, 'exp': datetime.utcnow() + timedelta(hours=1)}, private_key, algorithm='RS256')
+        register_token = jwt.encode({'register': True, 'exp': datetime.now(UTC).replace(tzinfo=None) + timedelta(hours=1)}, private_key, algorithm='RS256')
         cloudlog.info("getting pilotauth")
         resp = api_get("v2/pilotauth/", method='POST', timeout=15,
                        imei=imei1, imei2=imei2, serial=serial, public_key=public_key, register_token=register_token)

system/manager/manager.py

@@ -42,7 +42,7 @@ def manager_init() -> None:
     ("LongitudinalPersonality", str(log.LongitudinalPersonality.standard)),
   ]
   if not PC:
-    default_params.append(("LastUpdateTime", datetime.datetime.utcnow().isoformat().encode('utf8')))
+    default_params.append(("LastUpdateTime", datetime.datetime.now(datetime.UTC).replace(tzinfo=None).isoformat().encode('utf8')))
 
   if params.get_bool("RecordFrontLock"):
     params.put_bool("RecordFront", True)

system/qcomgpsd/qcomgpsd.py

@@ -173,7 +173,7 @@ def setup_quectel(diag: ModemDiag) -> bool:
     os.remove(ASSIST_DATA_FILE)
   #at_cmd("AT+QGPSXTRADATA?")
   if system_time_valid():
-    time_str = datetime.datetime.utcnow().strftime("%Y/%m/%d,%H:%M:%S")
+    time_str = datetime.datetime.now(datetime.UTC).replace(tzinfo=None).strftime("%Y/%m/%d,%H:%M:%S")
     at_cmd(f"AT+QGPSXTRATIME=0,\"{time_str}\",1,1,1000")
 
   at_cmd("AT+QGPSCFG=\"outport\",\"usbnmea\"")

system/qcomgpsd/tests/test_qcomgpsd.py

@@ -85,7 +85,7 @@ class TestRawgpsd:
     if should_be_loaded:
       assert valid_duration == "10080"  # should be max time
       injected_time = datetime.datetime.strptime(injected_time_str.replace("\"", ""), "%Y/%m/%d,%H:%M:%S")
-      assert abs((datetime.datetime.utcnow() - injected_time).total_seconds()) < 60*60*12
+      assert abs((datetime.datetime.now(datetime.UTC).replace(tzinfo=None) - injected_time).total_seconds()) < 60*60*12
     else:
       valid_duration, injected_time_str = out.split(",", 1)
       injected_time_str = injected_time_str.replace('\"', '').replace('\'', '')

system/statsd.py

@@ -133,7 +133,7 @@ def main() -> NoReturn:
       # flush when started state changes or after FLUSH_TIME_S
       if (time.monotonic() > last_flush_time + STATS_FLUSH_TIME_S) or (sm['deviceState'].started != started_prev):
         result = ""
-        current_time = datetime.utcnow().replace(tzinfo=UTC)
+        current_time = datetime.now(UTC)
         tags['started'] = sm['deviceState'].started
 
         for key, value in gauges.items():

system/ubloxd/pigeond.py

@@ -6,7 +6,7 @@ import serial
 import struct
 import requests
 import urllib.parse
-from datetime import datetime
+from datetime import datetime, UTC
 
 from cereal import messaging
 from openpilot.common.params import Params
@@ -196,7 +196,7 @@ def initialize_pigeon(pigeon: TTYPigeon) -> bool:
         cloudlog.error(f"failed to restore almanac backup, status: {restore_status}")
 
       # sending time to ublox
-      t_now = datetime.utcnow()
+      t_now = datetime.now(UTC).replace(tzinfo=None)
       if t_now >= datetime(2021, 6, 1):
         cloudlog.warning("Sending current time to ublox")
 

system/updated/updated.py

@@ -60,7 +60,7 @@ class WaitTimeHelper:
     self.ready_event.wait(timeout=t)
 
 def write_time_to_param(params, param) -> None:
-  t = datetime.datetime.utcnow()
+  t = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
   params.put(param, t.isoformat().encode('utf8'))
 
 def read_time_from_param(params, param) -> datetime.datetime | None:
@@ -279,7 +279,7 @@ class Updater:
     if len(self.branches):
       self.params.put("UpdaterAvailableBranches", ','.join(self.branches.keys()))
 
-    last_update = datetime.datetime.utcnow()
+    last_update = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
     if update_success:
       write_time_to_param(self.params, "LastUpdateTime")
     else:
@@ -323,7 +323,7 @@ class Updater:
     for alert in ("Offroad_UpdateFailed", "Offroad_ConnectivityNeeded", "Offroad_ConnectivityNeededPrompt"):
       set_offroad_alert(alert, False)
 
-    now = datetime.datetime.utcnow()
+    now = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
     dt = now - last_update
     build_metadata = get_build_metadata()
     if failed_count > 15 and exception is not None and self.has_internet:
@@ -429,7 +429,7 @@ def main() -> None:
       cloudlog.event("update installed")
 
     if not params.get("InstallDate"):
-      t = datetime.datetime.utcnow().isoformat()
+      t = datetime.datetime.now(datetime.UTC).replace(tzinfo=None).isoformat()
       params.put("InstallDate", t.encode('utf8'))
 
     updater = Updater()
@@ -469,7 +469,7 @@ def main() -> None:
 
         # download update
         last_fetch = read_time_from_param(params, "UpdaterLastFetchTime")
-        timed_out = last_fetch is None or (datetime.datetime.utcnow() - last_fetch > datetime.timedelta(days=3))
+        timed_out = last_fetch is None or (datetime.datetime.now(datetime.UTC).replace(tzinfo=None) - last_fetch > datetime.timedelta(days=3))
         user_requested_fetch = wait_helper.user_request == UserRequest.FETCH
         if params.get_bool("NetworkMetered") and not timed_out and not user_requested_fetch:
           cloudlog.info("skipping fetch, connection metered")

system/webrtc/tests/test_stream_session.py

@@ -3,6 +3,7 @@ import json
 # for aiortc and its dependencies
 import warnings
 warnings.filterwarnings("ignore", category=DeprecationWarning)
+warnings.filterwarnings("ignore", category=RuntimeWarning) # TODO: remove this when google-crc32c publish a python3.12 wheel
 
 from aiortc import RTCDataChannel
 from aiortc.mediastreams import VIDEO_CLOCK_RATE, VIDEO_TIME_BASE

system/webrtc/tests/test_webrtcd.py

@@ -4,6 +4,7 @@ import json
 # for aiortc and its dependencies
 import warnings
 warnings.filterwarnings("ignore", category=DeprecationWarning)
+warnings.filterwarnings("ignore", category=RuntimeWarning) # TODO: remove this when google-crc32c publish a python3.12 wheel
 
 from openpilot.system.webrtc.webrtcd import get_stream
 

system/webrtc/webrtcd.py

@@ -11,6 +11,7 @@ from typing import Any, TYPE_CHECKING
 # aiortc and its dependencies have lots of internal warnings :(
 import warnings
 warnings.filterwarnings("ignore", category=DeprecationWarning)
+warnings.filterwarnings("ignore", category=RuntimeWarning) # TODO: remove this when google-crc32c publish a python3.12 wheel
 
 import capnp
 from aiohttp import web

tools/lib/azure_container.py

@@ -1,5 +1,5 @@
 import os
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, UTC
 from functools import lru_cache
 from pathlib import Path
 from typing import IO
@@ -20,7 +20,7 @@ def get_azure_credential():
 @lru_cache
 def get_container_sas(account_name: str, container_name: str):
   from azure.storage.blob import BlobServiceClient, ContainerSasPermissions, generate_container_sas
-  start_time = datetime.utcnow()
+  start_time = datetime.now(UTC).replace(tzinfo=None)
   expiry_time = start_time + timedelta(hours=1)
   blob_service = BlobServiceClient(
     account_url=f"https://{account_name}.blob.core.windows.net",