mawei
/
openpilot0
mirror of https://github.com/commaai/openpilot.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

switch to uv (#32853) 

* switch to uv

* pin back numpy

* remove lock

* try without cache

* test with no cache

* no cache for all

* debug

* remove debug

* remove pygame dep

* Update README.md

* remove lower bound

* update repo maintenance job

* upgradable

* repo maintenance

* copy poetry api

* build system

* unpin pygame

* readd cache

* readd cache

* basic command

* lower delay

* macos

* fix authors

* macos fix

* use uv

* preview

* other way

* new metadrive

* uv sync

* cp lock in docker

* fix docker

* use hatchling

* fix devcontainer

* fix container

* path

* uv run

* cleanup

* pre-commit

* format

* release_file
This commit is contained in:
Maxime Desroches 2024-06-27 17:04:05 -07:00 committed by GitHub
parent 4a70934625
commit 12d5afba73
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 7997 additions and 8157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.devcontainer/Dockerfile
View File

@ -1,7 +1,8 @@
FROM ghcr.io/commaai/openpilot-base:latest
RUN apt update && apt install -y vim net-tools usbutils htop ripgrep tmux wget mesa-utils xvfb libxtst6 libxv1 libglu1-mesa gdb bash-completion
RUN pip install ipython jupyter jupyterlab
RUN python3 -m ensurepip --upgrade
RUN pip3 install ipython jupyter jupyterlab
RUN cd /tmp && \
ARCH=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && \

6
.github/workflows/repo-maintenance.yaml vendored
View File

@ -41,10 +41,10 @@ jobs:
if: github.repository == 'commaai/openpilot'
steps:
- uses: actions/checkout@v4
- name: poetry lock
- name: uv lock
run: |
pip install poetry
poetry lock
pip install uv
uv lock
- name: pre-commit autoupdate
run: |
git config --global --add safe.directory '*'

3
.github/workflows/selfdrive_tests.yaml vendored
View File

@ -62,7 +62,6 @@ jobs:
cd $GITHUB_WORKSPACE
cp .pre-commit-config.yaml $STRIPPED_DIR
cp pyproject.toml $STRIPPED_DIR
cp poetry.lock $STRIPPED_DIR
cd $STRIPPED_DIR
${{ env.RUN }} "unset PYTHONWARNINGS && SKIP=check-added-large-files,check-hooks-apply,check-useless-excludes pre-commit run --all && chmod -R 777 /tmp/pre-commit"
@ -102,7 +101,7 @@ jobs:
# package install has DeprecationWarnings
PYTHONWARNINGS: default
- name: Test openpilot environment
run: poetry run scons -h
run: . .venv/bin/activate && scons -h
static_analysis:
name: static analysis

10
.github/workflows/tools_tests.yaml vendored
View File

@ -53,10 +53,8 @@ jobs:
python-version: '3.11.4'
- name: Installing pip
run: pip install pip==24.0
- name: Installing poetry
run: pip install poetry==1.7.0
- name: Installing python dependencies
run: poetry install --no-cache --no-root
- name: Installing uv
run: pip install uv
- name: git LFS
run: git lfs pull
- run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
@ -69,7 +67,7 @@ jobs:
scons-${{ runner.arch }}-ubuntu2004-${{ env.CACHE_COMMIT_DATE }}
scons-${{ runner.arch }}-ubuntu2004
- name: Building openpilot
run: poetry run scons -u -j$(nproc)
run: uv run scons -u -j$(nproc)
- name: Saving scons cache
uses: actions/cache/save@v4
if: github.ref == 'refs/heads/master'
@ -100,6 +98,6 @@ jobs:
- name: Test environment
run: |
devcontainer exec --workspace-folder . scons -j$(nproc) cereal/ common/
devcontainer exec --workspace-folder . pip install pip-install-test
devcontainer exec --workspace-folder . pip3 install pip-install-test
devcontainer exec --workspace-folder . touch /home/batman/.comma/auth.json
devcontainer exec --workspace-folder . sudo touch /root/test.txt

11
.pre-commit-config.yaml
View File

@ -19,7 +19,7 @@ repos:
- id: check-executables-have-shebangs
- id: check-shebang-scripts-are-executable
- id: check-added-large-files
exclude: '(docs/CARS.md)|(poetry.lock)|(third_party/acados/include/blasfeo/include/blasfeo_d_kernel.h)'
exclude: '(docs/CARS.md)|(uv.lock)|(third_party/acados/include/blasfeo/include/blasfeo_d_kernel.h)'
args:
- --maxkb=120
- --enforce-all
@ -27,7 +27,7 @@ repos:
rev: v2.3.0
hooks:
- id: codespell
exclude: '^(third_party/)|(body/)|(msgq/)|(panda/)|(opendbc/)|(rednose/)|(rednose_repo/)|(teleoprtc/)|(teleoprtc_repo/)|(selfdrive/ui/translations/.*.ts)|(poetry.lock)'
exclude: '^(third_party/)|(body/)|(msgq/)|(panda/)|(opendbc/)|(rednose/)|(rednose_repo/)|(teleoprtc/)|(teleoprtc_repo/)|(selfdrive/ui/translations/.*.ts)|(uv.lock)'
args:
# if you've got a short variable name that's getting flagged, add it here
- -L bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie,preints,whit,indexIn
@ -92,13 +92,6 @@ repos:
language: system
pass_filenames: false
files: '^selfdrive/ui/translations/'
- repo: https://github.com/python-poetry/poetry
rev: '1.8.0'
hooks:
- id: poetry-check
name: validate poetry lock
args:
- --lock
- repo: https://github.com/python-jsonschema/check-jsonschema
rev: 0.28.5
hooks:

11
Dockerfile.openpilot_base
View File

@ -68,16 +68,15 @@ RUN usermod -aG sudo $USER
RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER $USER
COPY --chown=$USER pyproject.toml poetry.lock /tmp/
COPY --chown=$USER pyproject.toml uv.lock /tmp/
COPY --chown=$USER tools/install_python_dependencies.sh /tmp/tools/
ENV POETRY_VIRTUALENVS_CREATE=false
ENV VIRTUAL_ENV=/home/$USER/venv/.venv
RUN python3 -m venv $VIRTUAL_ENV
ENV VIRTUAL_ENV=/home/$USER/.venv
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
RUN . $VIRTUAL_ENV/bin/activate && \
cd /tmp && \
RUN cd /tmp && \
tools/install_python_dependencies.sh && \
mkdir -p $VIRTUAL_ENV && \
cp -r /tmp/.venv/* $VIRTUAL_ENV && \
rm -rf /tmp/* && \
rm -rf /home/$USER/.cache

7976
poetry.lock generated
View File

File diff suppressed because one or more lines are too long

255
pyproject.toml
View File

@ -1,12 +1,140 @@
[project]
name = "openpilot"
requires-python = ">= 3.11"
readme = "README.md"
license = {text = "MIT License"}
version = "0.1.0"
description = "an open source driver assistance system"
authors = [
{name ="Vehicle Researcher", email="user@comma.ai"}
]
dependencies = [
# multiple users
"sounddevice", # micd + soundd
"pyserial", # pigeond + qcomgpsd
"requests", # many one-off uses
"sympy", # rednose + friends
"crcmod", # cars + qcomgpsd
"tqdm", # cars (fw_versions.py) on start + many one-off uses
# hardwared
"smbus2", # configuring amp
# core
"cffi",
"scons",
"pycapnp",
"Cython",
"setuptools",
"numpy < 2.0.0", # control does not support numpy 2
# body / webrtcd
"aiohttp",
"aiortc",
"pyaudio",
# panda
"libusb1",
"spidev; platform_system == 'Linux'",
# modeld
"onnx >= 1.14.0",
"onnxruntime >=1.16.3; platform_system == 'Linux' and platform_machine == 'aarch64'",
"onnxruntime-gpu >=1.16.3; platform_system == 'Linux' and platform_machine == 'x86_64'",
# logging
"pyzmq",
"sentry-sdk",
# athena
"PyJWT",
"json-rpc",
"websocket_client",
# acados deps
"casadi",
"future-fstrings",
# these should be removed
"psutil",
"pycryptodome", # used in updated/casync, panda, body, and a test
#logreader
"zstd",
]
[project.optional-dependencies]
docs = [
"Jinja2",
"sphinx",
"sphinx-rtd-theme",
"sphinx-sitemap"
]
testing = [
"coverage",
"hypothesis ==6.47.*",
"mypy",
"pre-commit",
"pytest",
"pytest-cov",
"pytest-cpp",
"pytest-subtests",
"pytest-xdist",
"pytest-timeout",
"pytest-randomly",
"pytest-asyncio",
"pytest-mock",
"pytest-repeat",
"ruff"
]
dev = [
"av",
"azure-identity",
"azure-storage-blob",
"breathe",
"control",
"dictdiffer",
"flaky",
"inputs",
"lru-dict",
"matplotlib",
"metadrive-simulator; platform_machine != 'aarch64'",
"mpld3",
"myst-parser",
"natsort",
"opencv-python-headless",
"parameterized >=0.8, <0.9",
#pprofile = "*"
"pyautogui",
"pygame",
"pyopencl; platform_machine != 'aarch64'", # broken on arm64
"pywinctl",
"pyprof2calltree",
"rerun-sdk",
"tabulate",
"types-requests",
"types-tabulate",
# this is only pinned since 5.15.11 is broken
"pyqt5 ==5.15.2; platform_machine == 'x86_64'", # no aarch64 wheels for macOS/linux
]
[tool.uv.sources]
metadrive-simulator = { git = "https://github.com/commaai/metadrive.git", branch = "opencv_headless" }
[project.urls]
Homepage = "https://comma.ai"
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = [ "." ]
[tool.pytest.ini_options]
minversion = "6.0"
addopts = "--ignore=openpilot/ --ignore=cereal/ --ignore=opendbc/ --ignore=panda/ --ignore=rednose_repo/ --ignore=tinygrad_repo/ --ignore=teleoprtc_repo/ -Werror --strict-config --strict-markers --durations=10 -n auto --dist=loadgroup"
@ -75,131 +203,6 @@ warn_return_any=true
# allow implicit optionals for default args
implicit_optional = true
[tool.poetry]
name = "openpilot"
version = "0.1.0"
description = "an open source driver assistance system"
authors = ["Vehicle Researcher <user@comma.ai>"]
license = "MIT"
readme = "README.md"
repository = "https://github.com/commaai/openpilot"
documentation = "https://docs.comma.ai"
[tool.poetry.dependencies]
python = ">=3.11, <3.13"
# multiple users
sounddevice = "*" # micd + soundd
pyserial = "*" # pigeond + qcomgpsd
requests = "*" # many one-off uses
sympy = "*" # rednose + friends
crcmod = "*" # cars + qcomgpsd
tqdm = "*" # cars (fw_versions.py) on start + many one-off uses
# hardwared
smbus2 = "*" # configuring amp
# core
cffi = "*"
scons = "*"
pycapnp = "*"
Cython = "*"
setuptools = "*"
numpy = "*"
# body / webrtcd
aiohttp = "*"
aiortc = "*"
pyaudio = "*"
# panda
libusb1 = "*"
spidev = { version = "*", platform = "linux" }
# modeld
onnx = ">=1.14.0"
onnxruntime = { version = ">=1.16.3", platform = "linux", markers = "platform_machine == 'aarch64'" }
onnxruntime-gpu = { version = ">=1.16.3", platform = "linux", markers = "platform_machine == 'x86_64'" }
# logging
pyzmq = "*"
sentry-sdk = "*"
# athena
PyJWT = "*"
json-rpc = "*"
websocket_client = "*"
# acados deps
casadi = "*"
future-fstrings = "*"
# these should be removed
psutil = "*"
pycryptodome = "*" # used in updated/casync, panda, body, and a test
#logreader
zstd = "*"
[tool.poetry.group.docs.dependencies]
Jinja2 = "*"
sphinx = "*"
sphinx-rtd-theme = "*"
sphinx-sitemap = "*"
[tool.poetry.group.testing.dependencies]
coverage = "*"
hypothesis = "~6.47"
mypy = "*"
pre-commit = "*"
pytest = "*"
pytest-cov = "*"
pytest-cpp = "*"
pytest-subtests = "*"
pytest-xdist = "*"
pytest-timeout = "*"
pytest-randomly = "*"
pytest-asyncio = "*"
pytest-mock = "*"
pytest-repeat = "*"
ruff = "*"
[tool.poetry.group.dev.dependencies]
av = "*"
azure-identity = "*"
azure-storage-blob = "*"
breathe = "*"
control = "*"
dictdiffer = "*"
flaky = "*"
inputs = "*"
lru-dict = "*"
matplotlib = "*"
metadrive-simulator = { git = "https://github.com/commaai/metadrive.git", branch = "anisotropic_off", markers = "platform_machine != 'aarch64'" } # no linux/aarch64 wheels for certain dependencies
mpld3 = "*"
myst-parser = "*"
natsort = "*"
opencv-python-headless = "*"
parameterized = "^0.8"
#pprofile = "*"
pyautogui = "*"
pygame = "*"
pyopencl = { version = "*", markers = "platform_machine != 'aarch64'" } # broken on arm64
pywinctl = "*"
pyprof2calltree = "*"
rerun-sdk = "*"
tabulate = "*"
types-requests = "*"
types-tabulate = "*"
# this is only pinned since 5.15.11 is broken
pyqt5 = { version = "==5.15.2", markers = "platform_machine == 'x86_64'" } # no aarch64 wheels for macOS/linux
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
# https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml
[tool.ruff]
indent-width = 2

2
release/release_files.py
View File

@ -23,7 +23,7 @@ blacklist = [
"^common/tests/",
# particularly large text files
"poetry.lock",
"uv.lock",
"third_party/catch2",
"selfdrive/car/tests/test_models.*",

15
tools/README.md
View File

@ -26,16 +26,23 @@ git clone --recurse-submodules https://github.com/commaai/openpilot.git
``` bash
cd openpilot
git lfs pull
tools/ubuntu_setup.sh
```
Activate a shell with the Python dependencies installed:
**3. Git LFS**
``` bash
poetry shell
git lfs pull
```
**3. Build openpilot**
**4. Activate a python shell**
Activate a shell with the Python dependencies installed:
``` bash
source .venv/bin/activate
```
**5. Build openpilot**
``` bash
scons -u -j$(nproc)

32
tools/install_python_dependencies.sh
View File

@ -8,22 +8,17 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
ROOT=$DIR/../
cd $ROOT
RC_FILE="${HOME}/.$(basename ${SHELL})rc"
if [ "$(uname)" == "Darwin" ] && [ $SHELL == "/bin/bash" ]; then
RC_FILE="$HOME/.bash_profile"
fi
if ! command -v "poetry" > /dev/null 2>&1; then
echo "installing poetry..."
curl -sSL https://install.python-poetry.org | python3 -
POETRY_BIN='$HOME/.local/bin'
ADD_PATH_CMD="export PATH=\"$POETRY_BIN:\$PATH\""
if ! command -v "uv" > /dev/null 2>&1; then
echo "installing uv..."
curl -LsSf https://astral.sh/uv/install.sh | sh
UV_BIN='$HOME/.cargo/env'
ADD_PATH_CMD=". \"$UV_BIN\""
eval $ADD_PATH_CMD
printf "\n#poetry path\n$ADD_PATH_CMD\n" >> $RC_FILE
fi
poetry config virtualenvs.prefer-active-python true --local
poetry config virtualenvs.in-project true --local
echo "installing python packages..."
uv sync --all-extras
source .venv/bin/activate
echo "PYTHONPATH=${PWD}" > $ROOT/.env
if [[ "$(uname)" == 'Darwin' ]]; then
@ -32,15 +27,8 @@ if [[ "$(uname)" == 'Darwin' ]]; then
echo "export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES" >> $ROOT/.env
fi
poetry self add poetry-dotenv-plugin@^0.1.0
echo "installing python packages..."
poetry install --no-cache --no-root
[ -n "$POETRY_VIRTUALENVS_CREATE" ] && RUN="" || RUN="poetry run"
if [ "$(uname)" != "Darwin" ] && [ -e "$ROOT/.git" ]; then
echo "pre-commit hooks install..."
$RUN pre-commit install
$RUN git submodule foreach pre-commit install
pre-commit install
git submodule foreach pre-commit install
fi

7828
uv.lock Normal file
View File

File diff suppressed because it is too large Load Diff