switch to uv (#32853)

* switch to uv

* pin back numpy

* remove lock

* try without cache

* test with no cache

* no cache for all

* debug

* remove debug

* remove pygame dep

* Update README.md

* remove lower bound

* update repo maintenance job

* upgradable

* repo maintenance

* copy poetry api

* build system

* unpin pygame

* readd cache

* readd cache

* basic command

* lower delay

* macos

* fix authors

* macos fix

* use uv

* preview

* other way

* new metadrive

* uv sync

* cp lock in docker

* fix docker

* use hatchling

* fix devcontainer

* fix container

* path

* uv run

* cleanup

* pre-commit

* format

* release_file
old-commit-hash: 12d5afba73

.devcontainer/Dockerfile

@@ -1,7 +1,8 @@
 FROM ghcr.io/commaai/openpilot-base:latest
 
 RUN apt update && apt install -y vim net-tools usbutils htop ripgrep tmux wget mesa-utils xvfb libxtst6 libxv1 libglu1-mesa gdb bash-completion
-RUN pip install ipython jupyter jupyterlab
+RUN python3 -m ensurepip --upgrade
+RUN pip3 install ipython jupyter jupyterlab
 
 RUN cd /tmp && \
     ARCH=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && \

.github/workflows/repo-maintenance.yaml

@@ -41,10 +41,10 @@ jobs:
     if: github.repository == 'commaai/openpilot'
     steps:
     - uses: actions/checkout@v4
-    - name: poetry lock
+    - name: uv lock
       run: |
-        pip install poetry
-        poetry lock
+        pip install uv
+        uv lock
     - name: pre-commit autoupdate
       run: |
         git config --global --add safe.directory '*'

.github/workflows/selfdrive_tests.yaml

@@ -62,7 +62,6 @@ jobs:
         cd $GITHUB_WORKSPACE
         cp .pre-commit-config.yaml $STRIPPED_DIR
         cp pyproject.toml $STRIPPED_DIR
-        cp poetry.lock $STRIPPED_DIR
         cd $STRIPPED_DIR
         ${{ env.RUN }} "unset PYTHONWARNINGS && SKIP=check-added-large-files,check-hooks-apply,check-useless-excludes pre-commit run --all && chmod -R 777 /tmp/pre-commit"
 
@@ -86,7 +85,7 @@ jobs:
         docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
     - uses: ./.github/workflows/compile-openpilot
       timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 15 || 30) }} # allow more time when we missed the scons cache
-      
+
   build_mac:
     name: build macos
     runs-on: macos-latest
@@ -102,7 +101,7 @@ jobs:
         # package install has DeprecationWarnings
         PYTHONWARNINGS: default
     - name: Test openpilot environment
-      run: poetry run scons -h
+      run: . .venv/bin/activate && scons -h
 
   static_analysis:
     name: static analysis

.github/workflows/tools_tests.yaml

@@ -53,10 +53,8 @@ jobs:
         python-version: '3.11.4'
     - name: Installing pip
       run: pip install pip==24.0
-    - name: Installing poetry
-      run: pip install poetry==1.7.0
-    - name: Installing python dependencies
-      run: poetry install --no-cache --no-root
+    - name: Installing uv
+      run: pip install uv
     - name: git LFS
       run: git lfs pull
     - run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
@@ -69,7 +67,7 @@ jobs:
           scons-${{ runner.arch }}-ubuntu2004-${{ env.CACHE_COMMIT_DATE }}
           scons-${{ runner.arch }}-ubuntu2004
     - name: Building openpilot
-      run: poetry run scons -u -j$(nproc)
+      run: uv run scons -u -j$(nproc)
     - name: Saving scons cache
       uses: actions/cache/save@v4
       if: github.ref == 'refs/heads/master'
@@ -100,6 +98,6 @@ jobs:
     - name: Test environment
       run: |
         devcontainer exec --workspace-folder . scons -j$(nproc) cereal/ common/
-        devcontainer exec --workspace-folder . pip install pip-install-test
+        devcontainer exec --workspace-folder . pip3 install pip-install-test
         devcontainer exec --workspace-folder . touch /home/batman/.comma/auth.json
         devcontainer exec --workspace-folder . sudo touch /root/test.txt

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
     -   id: check-executables-have-shebangs
     -   id: check-shebang-scripts-are-executable
     -   id: check-added-large-files
-        exclude: '(docs/CARS.md)|(poetry.lock)|(third_party/acados/include/blasfeo/include/blasfeo_d_kernel.h)'
+        exclude: '(docs/CARS.md)|(uv.lock)|(third_party/acados/include/blasfeo/include/blasfeo_d_kernel.h)'
         args:
         - --maxkb=120
         - --enforce-all
@@ -27,7 +27,7 @@ repos:
     rev: v2.3.0
     hooks:
     -   id: codespell
-        exclude: '^(third_party/)|(body/)|(msgq/)|(panda/)|(opendbc/)|(rednose/)|(rednose_repo/)|(teleoprtc/)|(teleoprtc_repo/)|(selfdrive/ui/translations/.*.ts)|(poetry.lock)'
+        exclude: '^(third_party/)|(body/)|(msgq/)|(panda/)|(opendbc/)|(rednose/)|(rednose_repo/)|(teleoprtc/)|(teleoprtc_repo/)|(selfdrive/ui/translations/.*.ts)|(uv.lock)'
         args:
         # if you've got a short variable name that's getting flagged, add it here
         - -L bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie,preints,whit,indexIn
@@ -92,13 +92,6 @@ repos:
         language: system
         pass_filenames: false
         files: '^selfdrive/ui/translations/'
--   repo: https://github.com/python-poetry/poetry
-    rev: '1.8.0'
-    hooks:
-    -   id: poetry-check
-        name: validate poetry lock
-        args:
-        - --lock
 -   repo: https://github.com/python-jsonschema/check-jsonschema
     rev: 0.28.5
     hooks:

Dockerfile.openpilot_base

@@ -68,16 +68,15 @@ RUN usermod -aG sudo $USER
 RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 USER $USER
 
-COPY --chown=$USER pyproject.toml poetry.lock /tmp/
+COPY --chown=$USER pyproject.toml uv.lock /tmp/
 COPY --chown=$USER tools/install_python_dependencies.sh /tmp/tools/
 
-ENV POETRY_VIRTUALENVS_CREATE=false
-ENV VIRTUAL_ENV=/home/$USER/venv/.venv
-RUN python3 -m venv $VIRTUAL_ENV
+ENV VIRTUAL_ENV=/home/$USER/.venv
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-RUN . $VIRTUAL_ENV/bin/activate && \
-    cd /tmp && \
+RUN cd /tmp && \
     tools/install_python_dependencies.sh && \
+    mkdir -p $VIRTUAL_ENV && \
+    cp -r /tmp/.venv/* $VIRTUAL_ENV && \
     rm -rf /tmp/* && \
     rm -rf /home/$USER/.cache
 

poetry.lock

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:fd842a62ebdc85d58bf3dee17ff65b730f526f74e48ed124765b42233e173f58
-size 619932

pyproject.toml

@@ -1,12 +1,140 @@
 [project]
 name = "openpilot"
 requires-python = ">= 3.11"
-readme = "README.md"
 license = {text = "MIT License"}
+version = "0.1.0"
+description = "an open source driver assistance system"
+authors = [
+  {name ="Vehicle Researcher", email="user@comma.ai"}
+]
+
+dependencies = [
+  # multiple users
+  "sounddevice",  # micd + soundd
+  "pyserial",     # pigeond + qcomgpsd
+  "requests",     # many one-off uses
+  "sympy",        # rednose + friends
+  "crcmod",       # cars + qcomgpsd
+  "tqdm",         # cars (fw_versions.py) on start + many one-off uses
+
+  # hardwared
+  "smbus2", # configuring amp
+
+  # core
+  "cffi",
+  "scons",
+  "pycapnp",
+  "Cython",
+  "setuptools",
+  "numpy < 2.0.0", # control does not support numpy 2
+
+  # body / webrtcd
+  "aiohttp",
+  "aiortc",
+  "pyaudio",
+
+  # panda
+  "libusb1",
+  "spidev; platform_system == 'Linux'",
+
+  # modeld
+  "onnx >= 1.14.0",
+  "onnxruntime >=1.16.3; platform_system == 'Linux' and platform_machine == 'aarch64'",
+  "onnxruntime-gpu >=1.16.3; platform_system == 'Linux' and platform_machine == 'x86_64'",
+
+  # logging
+  "pyzmq",
+  "sentry-sdk",
+
+  # athena
+  "PyJWT",
+  "json-rpc",
+  "websocket_client",
+
+  # acados deps
+  "casadi",
+  "future-fstrings",
+
+  # these should be removed
+  "psutil",
+  "pycryptodome", # used in updated/casync, panda, body, and a test
+
+  #logreader
+  "zstd",
+]
+
+[project.optional-dependencies]
+docs = [
+  "Jinja2",
+  "sphinx",
+  "sphinx-rtd-theme",
+  "sphinx-sitemap"
+]
+
+testing = [
+  "coverage",
+  "hypothesis ==6.47.*",
+  "mypy",
+  "pre-commit",
+  "pytest",
+  "pytest-cov",
+  "pytest-cpp",
+  "pytest-subtests",
+  "pytest-xdist",
+  "pytest-timeout",
+  "pytest-randomly",
+  "pytest-asyncio",
+  "pytest-mock",
+  "pytest-repeat",
+  "ruff"
+]
+
+dev = [
+  "av",
+  "azure-identity",
+  "azure-storage-blob",
+  "breathe",
+  "control",
+  "dictdiffer",
+  "flaky",
+  "inputs",
+  "lru-dict",
+  "matplotlib",
+  "metadrive-simulator; platform_machine != 'aarch64'",
+  "mpld3",
+  "myst-parser",
+  "natsort",
+  "opencv-python-headless",
+  "parameterized >=0.8, <0.9",
+  #pprofile = "*"
+  "pyautogui",
+  "pygame",
+  "pyopencl; platform_machine != 'aarch64'", # broken on arm64
+  "pywinctl",
+  "pyprof2calltree",
+  "rerun-sdk",
+  "tabulate",
+  "types-requests",
+  "types-tabulate",
+
+  # this is only pinned since 5.15.11 is broken
+  "pyqt5 ==5.15.2; platform_machine == 'x86_64'", # no aarch64 wheels for macOS/linux
+
+]
+
+[tool.uv.sources]
+metadrive-simulator = { git = "https://github.com/commaai/metadrive.git", branch = "opencv_headless" }
 
 [project.urls]
 Homepage = "https://comma.ai"
 
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = [ "." ]
+
 [tool.pytest.ini_options]
 minversion = "6.0"
 addopts = "--ignore=openpilot/ --ignore=cereal/ --ignore=opendbc/ --ignore=panda/ --ignore=rednose_repo/ --ignore=tinygrad_repo/ --ignore=teleoprtc_repo/ -Werror --strict-config --strict-markers --durations=10 -n auto --dist=loadgroup"
@@ -75,131 +203,6 @@ warn_return_any=true
 # allow implicit optionals for default args
 implicit_optional = true
 
-
-[tool.poetry]
-name = "openpilot"
-version = "0.1.0"
-description = "an open source driver assistance system"
-authors = ["Vehicle Researcher <user@comma.ai>"]
-license = "MIT"
-readme = "README.md"
-repository = "https://github.com/commaai/openpilot"
-documentation = "https://docs.comma.ai"
-
-[tool.poetry.dependencies]
-python = ">=3.11, <3.13"
-
-# multiple users
-sounddevice = "*"  # micd + soundd
-pyserial = "*"     # pigeond + qcomgpsd
-requests = "*"     # many one-off uses
-sympy = "*"        # rednose + friends
-crcmod = "*"       # cars + qcomgpsd
-tqdm = "*"         # cars (fw_versions.py) on start + many one-off uses
-
-# hardwared
-smbus2 = "*"  # configuring amp
-
-# core
-cffi = "*"
-scons = "*"
-pycapnp = "*"
-Cython = "*"
-setuptools = "*"
-numpy = "*"
-
-# body / webrtcd
-aiohttp = "*"
-aiortc = "*"
-pyaudio = "*"
-
-# panda
-libusb1 = "*"
-spidev = { version = "*", platform = "linux" }
-
-# modeld
-onnx = ">=1.14.0"
-onnxruntime = { version = ">=1.16.3", platform = "linux", markers = "platform_machine == 'aarch64'" }
-onnxruntime-gpu = { version = ">=1.16.3", platform = "linux", markers = "platform_machine == 'x86_64'" }
-
-# logging
-pyzmq = "*"
-sentry-sdk = "*"
-
-# athena
-PyJWT = "*"
-json-rpc = "*"
-websocket_client = "*"
-
-# acados deps
-casadi = "*"
-future-fstrings = "*"
-
-# these should be removed
-psutil = "*"
-pycryptodome = "*"  # used in updated/casync, panda, body, and a test
-
-#logreader
-zstd = "*"
-
-[tool.poetry.group.docs.dependencies]
-Jinja2 = "*"
-sphinx = "*"
-sphinx-rtd-theme = "*"
-sphinx-sitemap = "*"
-
-[tool.poetry.group.testing.dependencies]
-coverage = "*"
-hypothesis = "~6.47"
-mypy = "*"
-pre-commit = "*"
-pytest = "*"
-pytest-cov = "*"
-pytest-cpp = "*"
-pytest-subtests = "*"
-pytest-xdist = "*"
-pytest-timeout = "*"
-pytest-randomly = "*"
-pytest-asyncio = "*"
-pytest-mock = "*"
-pytest-repeat = "*"
-ruff = "*"
-
-[tool.poetry.group.dev.dependencies]
-av = "*"
-azure-identity = "*"
-azure-storage-blob = "*"
-breathe = "*"
-control = "*"
-dictdiffer = "*"
-flaky = "*"
-inputs = "*"
-lru-dict = "*"
-matplotlib = "*"
-metadrive-simulator = { git = "https://github.com/commaai/metadrive.git", branch = "anisotropic_off", markers = "platform_machine != 'aarch64'" } # no linux/aarch64 wheels for certain dependencies
-mpld3 = "*"
-myst-parser = "*"
-natsort = "*"
-opencv-python-headless = "*"
-parameterized = "^0.8"
-#pprofile = "*"
-pyautogui = "*"
-pygame = "*"
-pyopencl = { version = "*", markers = "platform_machine != 'aarch64'" }  # broken on arm64
-pywinctl = "*"
-pyprof2calltree = "*"
-rerun-sdk = "*"
-tabulate = "*"
-types-requests = "*"
-types-tabulate = "*"
-
-# this is only pinned since 5.15.11 is broken
-pyqt5 = { version = "==5.15.2", markers = "platform_machine == 'x86_64'" } # no aarch64 wheels for macOS/linux
-
-[build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"
-
 # https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml
 [tool.ruff]
 indent-width = 2

release/release_files.py

@@ -23,7 +23,7 @@ blacklist = [
   "^common/tests/",
 
   # particularly large text files
-  "poetry.lock",
+  "uv.lock",
   "third_party/catch2",
   "selfdrive/car/tests/test_models.*",
 

tools/README.md

@@ -26,16 +26,23 @@ git clone --recurse-submodules https://github.com/commaai/openpilot.git
 
 ``` bash
 cd openpilot
-git lfs pull
 tools/ubuntu_setup.sh
 ```
 
-Activate a shell with the Python dependencies installed:
+**3. Git LFS**
+
 ``` bash
-poetry shell
+git lfs pull
 ```
 
-**3. Build openpilot**
+**4. Activate a python shell**
+
+Activate a shell with the Python dependencies installed:
+``` bash
+source .venv/bin/activate
+```
+
+**5. Build openpilot**
 
 ``` bash
 scons -u -j$(nproc)

tools/install_python_dependencies.sh

@@ -8,22 +8,17 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 ROOT=$DIR/../
 cd $ROOT
 
-RC_FILE="${HOME}/.$(basename ${SHELL})rc"
-if [ "$(uname)" == "Darwin" ] && [ $SHELL == "/bin/bash" ]; then
-  RC_FILE="$HOME/.bash_profile"
-fi
-
-if ! command -v "poetry" > /dev/null 2>&1; then
-  echo "installing poetry..."
-  curl -sSL https://install.python-poetry.org | python3 -
-  POETRY_BIN='$HOME/.local/bin'
-  ADD_PATH_CMD="export PATH=\"$POETRY_BIN:\$PATH\""
+if ! command -v "uv" > /dev/null 2>&1; then
+  echo "installing uv..."
+  curl -LsSf https://astral.sh/uv/install.sh | sh
+  UV_BIN='$HOME/.cargo/env'
+  ADD_PATH_CMD=". \"$UV_BIN\""
   eval $ADD_PATH_CMD
-  printf "\n#poetry path\n$ADD_PATH_CMD\n" >> $RC_FILE
 fi
 
-poetry config virtualenvs.prefer-active-python true --local
-poetry config virtualenvs.in-project true --local
+echo "installing python packages..."
+uv sync --all-extras
+source .venv/bin/activate
 
 echo "PYTHONPATH=${PWD}" > $ROOT/.env
 if [[ "$(uname)" == 'Darwin' ]]; then
@@ -32,15 +27,8 @@ if [[ "$(uname)" == 'Darwin' ]]; then
   echo "export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES" >> $ROOT/.env
 fi
 
-poetry self add poetry-dotenv-plugin@^0.1.0
-
-echo "installing python packages..."
-poetry install --no-cache --no-root
-
-[ -n "$POETRY_VIRTUALENVS_CREATE" ] && RUN="" || RUN="poetry run"
-
 if [ "$(uname)" != "Darwin" ] && [ -e "$ROOT/.git" ]; then
   echo "pre-commit hooks install..."
-  $RUN pre-commit install
-  $RUN git submodule foreach pre-commit install
+  pre-commit install
+  git submodule foreach pre-commit install
 fi