mirror of https://github.com/1okko/openpilot.git
switch to uv (#32853)
* switch to uv * pin back numpy * remove lock * try without cache * test with no cache * no cache for all * debug * remove debug * remove pygame dep * Update README.md * remove lower bound * update repo maintenance job * upgradable * repo maintenance * copy poetry api * build system * unpin pygame * readd cache * readd cache * basic command * lower delay * macos * fix authors * macos fix * use uv * preview * other way * new metadrive * uv sync * cp lock in docker * fix docker * use hatchling * fix devcontainer * fix container * path * uv run * cleanup * pre-commit * format * release_file
This commit is contained in:
Maxime Desroches
GitHub
parent
4a70934625
commit
12d5afba73
|
|
@ -1,7 +1,8 @@
|
|||
FROM ghcr.io/commaai/openpilot-base:latest
|
||||
|
||||
RUN apt update && apt install -y vim net-tools usbutils htop ripgrep tmux wget mesa-utils xvfb libxtst6 libxv1 libglu1-mesa gdb bash-completion
|
||||
RUN pip install ipython jupyter jupyterlab
|
||||
RUN python3 -m ensurepip --upgrade
|
||||
RUN pip3 install ipython jupyter jupyterlab
|
||||
|
||||
RUN cd /tmp && \
|
||||
ARCH=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && \
|
||||
|
|
|
|||
|
|
@ -41,10 +41,10 @@ jobs:
|
|||
if: github.repository == 'commaai/openpilot'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: poetry lock
|
||||
- name: uv lock
|
||||
run: |
|
||||
pip install poetry
|
||||
poetry lock
|
||||
pip install uv
|
||||
uv lock
|
||||
- name: pre-commit autoupdate
|
||||
run: |
|
||||
git config --global --add safe.directory '*'
|
||||
|
|
|
|||
|
|
@ -62,7 +62,6 @@ jobs:
|
|||
cd $GITHUB_WORKSPACE
|
||||
cp .pre-commit-config.yaml $STRIPPED_DIR
|
||||
cp pyproject.toml $STRIPPED_DIR
|
||||
cp poetry.lock $STRIPPED_DIR
|
||||
cd $STRIPPED_DIR
|
||||
${{ env.RUN }} "unset PYTHONWARNINGS && SKIP=check-added-large-files,check-hooks-apply,check-useless-excludes pre-commit run --all && chmod -R 777 /tmp/pre-commit"
|
||||
|
||||
|
|
@ -86,7 +85,7 @@ jobs:
|
|||
docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
|
||||
- uses: ./.github/workflows/compile-openpilot
|
||||
timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 15 || 30) }} # allow more time when we missed the scons cache
|
||||
|
||||
|
||||
build_mac:
|
||||
name: build macos
|
||||
runs-on: macos-latest
|
||||
|
|
@ -102,7 +101,7 @@ jobs:
|
|||
# package install has DeprecationWarnings
|
||||
PYTHONWARNINGS: default
|
||||
- name: Test openpilot environment
|
||||
run: poetry run scons -h
|
||||
run: . .venv/bin/activate && scons -h
|
||||
|
||||
static_analysis:
|
||||
name: static analysis
|
||||
|
|
|
|||
|
|
@ -53,10 +53,8 @@ jobs:
|
|||
python-version: '3.11.4'
|
||||
- name: Installing pip
|
||||
run: pip install pip==24.0
|
||||
- name: Installing poetry
|
||||
run: pip install poetry==1.7.0
|
||||
- name: Installing python dependencies
|
||||
run: poetry install --no-cache --no-root
|
||||
- name: Installing uv
|
||||
run: pip install uv
|
||||
- name: git LFS
|
||||
run: git lfs pull
|
||||
- run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
|
||||
|
|
@ -69,7 +67,7 @@ jobs:
|
|||
scons-${{ runner.arch }}-ubuntu2004-${{ env.CACHE_COMMIT_DATE }}
|
||||
scons-${{ runner.arch }}-ubuntu2004
|
||||
- name: Building openpilot
|
||||
run: poetry run scons -u -j$(nproc)
|
||||
run: uv run scons -u -j$(nproc)
|
||||
- name: Saving scons cache
|
||||
uses: actions/cache/save@v4
|
||||
if: github.ref == 'refs/heads/master'
|
||||
|
|
@ -100,6 +98,6 @@ jobs:
|
|||
- name: Test environment
|
||||
run: |
|
||||
devcontainer exec --workspace-folder . scons -j$(nproc) cereal/ common/
|
||||
devcontainer exec --workspace-folder . pip install pip-install-test
|
||||
devcontainer exec --workspace-folder . pip3 install pip-install-test
|
||||
devcontainer exec --workspace-folder . touch /home/batman/.comma/auth.json
|
||||
devcontainer exec --workspace-folder . sudo touch /root/test.txt
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ repos:
|
|||
- id: check-executables-have-shebangs
|
||||
- id: check-shebang-scripts-are-executable
|
||||
- id: check-added-large-files
|
||||
exclude: '(docs/CARS.md)|(poetry.lock)|(third_party/acados/include/blasfeo/include/blasfeo_d_kernel.h)'
|
||||
exclude: '(docs/CARS.md)|(uv.lock)|(third_party/acados/include/blasfeo/include/blasfeo_d_kernel.h)'
|
||||
args:
|
||||
- --maxkb=120
|
||||
- --enforce-all
|
||||
|
|
@ -27,7 +27,7 @@ repos:
|
|||
rev: v2.3.0
|
||||
hooks:
|
||||
- id: codespell
|
||||
exclude: '^(third_party/)|(body/)|(msgq/)|(panda/)|(opendbc/)|(rednose/)|(rednose_repo/)|(teleoprtc/)|(teleoprtc_repo/)|(selfdrive/ui/translations/.*.ts)|(poetry.lock)'
|
||||
exclude: '^(third_party/)|(body/)|(msgq/)|(panda/)|(opendbc/)|(rednose/)|(rednose_repo/)|(teleoprtc/)|(teleoprtc_repo/)|(selfdrive/ui/translations/.*.ts)|(uv.lock)'
|
||||
args:
|
||||
# if you've got a short variable name that's getting flagged, add it here
|
||||
- -L bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie,preints,whit,indexIn
|
||||
|
|
@ -92,13 +92,6 @@ repos:
|
|||
language: system
|
||||
pass_filenames: false
|
||||
files: '^selfdrive/ui/translations/'
|
||||
- repo: https://github.com/python-poetry/poetry
|
||||
rev: '1.8.0'
|
||||
hooks:
|
||||
- id: poetry-check
|
||||
name: validate poetry lock
|
||||
args:
|
||||
- --lock
|
||||
- repo: https://github.com/python-jsonschema/check-jsonschema
|
||||
rev: 0.28.5
|
||||
hooks:
|
||||
|
|
|
|||
|
|
@ -68,16 +68,15 @@ RUN usermod -aG sudo $USER
|
|||
RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
|
||||
USER $USER
|
||||
|
||||
COPY --chown=$USER pyproject.toml poetry.lock /tmp/
|
||||
COPY --chown=$USER pyproject.toml uv.lock /tmp/
|
||||
COPY --chown=$USER tools/install_python_dependencies.sh /tmp/tools/
|
||||
|
||||
ENV POETRY_VIRTUALENVS_CREATE=false
|
||||
ENV VIRTUAL_ENV=/home/$USER/venv/.venv
|
||||
RUN python3 -m venv $VIRTUAL_ENV
|
||||
ENV VIRTUAL_ENV=/home/$USER/.venv
|
||||
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
|
||||
RUN . $VIRTUAL_ENV/bin/activate && \
|
||||
cd /tmp && \
|
||||
RUN cd /tmp && \
|
||||
tools/install_python_dependencies.sh && \
|
||||
mkdir -p $VIRTUAL_ENV && \
|
||||
cp -r /tmp/.venv/* $VIRTUAL_ENV && \
|
||||
rm -rf /tmp/* && \
|
||||
rm -rf /home/$USER/.cache
|
||||
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
255
pyproject.toml
255
pyproject.toml
|
|
@ -1,12 +1,140 @@
|
|||
[project]
|
||||
name = "openpilot"
|
||||
requires-python = ">= 3.11"
|
||||
readme = "README.md"
|
||||
license = {text = "MIT License"}
|
||||
version = "0.1.0"
|
||||
description = "an open source driver assistance system"
|
||||
authors = [
|
||||
{name ="Vehicle Researcher", email="user@comma.ai"}
|
||||
]
|
||||
|
||||
dependencies = [
|
||||
# multiple users
|
||||
"sounddevice", # micd + soundd
|
||||
"pyserial", # pigeond + qcomgpsd
|
||||
"requests", # many one-off uses
|
||||
"sympy", # rednose + friends
|
||||
"crcmod", # cars + qcomgpsd
|
||||
"tqdm", # cars (fw_versions.py) on start + many one-off uses
|
||||
|
||||
# hardwared
|
||||
"smbus2", # configuring amp
|
||||
|
||||
# core
|
||||
"cffi",
|
||||
"scons",
|
||||
"pycapnp",
|
||||
"Cython",
|
||||
"setuptools",
|
||||
"numpy < 2.0.0", # control does not support numpy 2
|
||||
|
||||
# body / webrtcd
|
||||
"aiohttp",
|
||||
"aiortc",
|
||||
"pyaudio",
|
||||
|
||||
# panda
|
||||
"libusb1",
|
||||
"spidev; platform_system == 'Linux'",
|
||||
|
||||
# modeld
|
||||
"onnx >= 1.14.0",
|
||||
"onnxruntime >=1.16.3; platform_system == 'Linux' and platform_machine == 'aarch64'",
|
||||
"onnxruntime-gpu >=1.16.3; platform_system == 'Linux' and platform_machine == 'x86_64'",
|
||||
|
||||
# logging
|
||||
"pyzmq",
|
||||
"sentry-sdk",
|
||||
|
||||
# athena
|
||||
"PyJWT",
|
||||
"json-rpc",
|
||||
"websocket_client",
|
||||
|
||||
# acados deps
|
||||
"casadi",
|
||||
"future-fstrings",
|
||||
|
||||
# these should be removed
|
||||
"psutil",
|
||||
"pycryptodome", # used in updated/casync, panda, body, and a test
|
||||
|
||||
#logreader
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
docs = [
|
||||
"Jinja2",
|
||||
"sphinx",
|
||||
"sphinx-rtd-theme",
|
||||
"sphinx-sitemap"
|
||||
]
|
||||
|
||||
testing = [
|
||||
"coverage",
|
||||
"hypothesis ==6.47.*",
|
||||
"mypy",
|
||||
"pre-commit",
|
||||
"pytest",
|
||||
"pytest-cov",
|
||||
"pytest-cpp",
|
||||
"pytest-subtests",
|
||||
"pytest-xdist",
|
||||
"pytest-timeout",
|
||||
"pytest-randomly",
|
||||
"pytest-asyncio",
|
||||
"pytest-mock",
|
||||
"pytest-repeat",
|
||||
"ruff"
|
||||
]
|
||||
|
||||
dev = [
|
||||
"av",
|
||||
"azure-identity",
|
||||
"azure-storage-blob",
|
||||
"breathe",
|
||||
"control",
|
||||
"dictdiffer",
|
||||
"flaky",
|
||||
"inputs",
|
||||
"lru-dict",
|
||||
"matplotlib",
|
||||
"metadrive-simulator; platform_machine != 'aarch64'",
|
||||
"mpld3",
|
||||
"myst-parser",
|
||||
"natsort",
|
||||
"opencv-python-headless",
|
||||
"parameterized >=0.8, <0.9",
|
||||
#pprofile = "*"
|
||||
"pyautogui",
|
||||
"pygame",
|
||||
"pyopencl; platform_machine != 'aarch64'", # broken on arm64
|
||||
"pywinctl",
|
||||
"pyprof2calltree",
|
||||
"rerun-sdk",
|
||||
"tabulate",
|
||||
"types-requests",
|
||||
"types-tabulate",
|
||||
|
||||
# this is only pinned since 5.15.11 is broken
|
||||
"pyqt5 ==5.15.2; platform_machine == 'x86_64'", # no aarch64 wheels for macOS/linux
|
||||
|
||||
]
|
||||
|
||||
[tool.uv.sources]
|
||||
metadrive-simulator = { git = "https://github.com/commaai/metadrive.git", branch = "opencv_headless" }
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://comma.ai"
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = [ "." ]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
minversion = "6.0"
|
||||
addopts = "--ignore=openpilot/ --ignore=cereal/ --ignore=opendbc/ --ignore=panda/ --ignore=rednose_repo/ --ignore=tinygrad_repo/ --ignore=teleoprtc_repo/ -Werror --strict-config --strict-markers --durations=10 -n auto --dist=loadgroup"
|
||||
|
|
@ -75,131 +203,6 @@ warn_return_any=true
|
|||
# allow implicit optionals for default args
|
||||
implicit_optional = true
|
||||
|
||||
|
||||
[tool.poetry]
|
||||
name = "openpilot"
|
||||
version = "0.1.0"
|
||||
description = "an open source driver assistance system"
|
||||
authors = ["Vehicle Researcher <user@comma.ai>"]
|
||||
license = "MIT"
|
||||
readme = "README.md"
|
||||
repository = "https://github.com/commaai/openpilot"
|
||||
documentation = "https://docs.comma.ai"
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = ">=3.11, <3.13"
|
||||
|
||||
# multiple users
|
||||
sounddevice = "*" # micd + soundd
|
||||
pyserial = "*" # pigeond + qcomgpsd
|
||||
requests = "*" # many one-off uses
|
||||
sympy = "*" # rednose + friends
|
||||
crcmod = "*" # cars + qcomgpsd
|
||||
tqdm = "*" # cars (fw_versions.py) on start + many one-off uses
|
||||
|
||||
# hardwared
|
||||
smbus2 = "*" # configuring amp
|
||||
|
||||
# core
|
||||
cffi = "*"
|
||||
scons = "*"
|
||||
pycapnp = "*"
|
||||
Cython = "*"
|
||||
setuptools = "*"
|
||||
numpy = "*"
|
||||
|
||||
# body / webrtcd
|
||||
aiohttp = "*"
|
||||
aiortc = "*"
|
||||
pyaudio = "*"
|
||||
|
||||
# panda
|
||||
libusb1 = "*"
|
||||
spidev = { version = "*", platform = "linux" }
|
||||
|
||||
# modeld
|
||||
onnx = ">=1.14.0"
|
||||
onnxruntime = { version = ">=1.16.3", platform = "linux", markers = "platform_machine == 'aarch64'" }
|
||||
onnxruntime-gpu = { version = ">=1.16.3", platform = "linux", markers = "platform_machine == 'x86_64'" }
|
||||
|
||||
# logging
|
||||
pyzmq = "*"
|
||||
sentry-sdk = "*"
|
||||
|
||||
# athena
|
||||
PyJWT = "*"
|
||||
json-rpc = "*"
|
||||
websocket_client = "*"
|
||||
|
||||
# acados deps
|
||||
casadi = "*"
|
||||
future-fstrings = "*"
|
||||
|
||||
# these should be removed
|
||||
psutil = "*"
|
||||
pycryptodome = "*" # used in updated/casync, panda, body, and a test
|
||||
|
||||
#logreader
|
||||
zstd = "*"
|
||||
|
||||
[tool.poetry.group.docs.dependencies]
|
||||
Jinja2 = "*"
|
||||
sphinx = "*"
|
||||
sphinx-rtd-theme = "*"
|
||||
sphinx-sitemap = "*"
|
||||
|
||||
[tool.poetry.group.testing.dependencies]
|
||||
coverage = "*"
|
||||
hypothesis = "~6.47"
|
||||
mypy = "*"
|
||||
pre-commit = "*"
|
||||
pytest = "*"
|
||||
pytest-cov = "*"
|
||||
pytest-cpp = "*"
|
||||
pytest-subtests = "*"
|
||||
pytest-xdist = "*"
|
||||
pytest-timeout = "*"
|
||||
pytest-randomly = "*"
|
||||
pytest-asyncio = "*"
|
||||
pytest-mock = "*"
|
||||
pytest-repeat = "*"
|
||||
ruff = "*"
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
av = "*"
|
||||
azure-identity = "*"
|
||||
azure-storage-blob = "*"
|
||||
breathe = "*"
|
||||
control = "*"
|
||||
dictdiffer = "*"
|
||||
flaky = "*"
|
||||
inputs = "*"
|
||||
lru-dict = "*"
|
||||
matplotlib = "*"
|
||||
metadrive-simulator = { git = "https://github.com/commaai/metadrive.git", branch = "anisotropic_off", markers = "platform_machine != 'aarch64'" } # no linux/aarch64 wheels for certain dependencies
|
||||
mpld3 = "*"
|
||||
myst-parser = "*"
|
||||
natsort = "*"
|
||||
opencv-python-headless = "*"
|
||||
parameterized = "^0.8"
|
||||
#pprofile = "*"
|
||||
pyautogui = "*"
|
||||
pygame = "*"
|
||||
pyopencl = { version = "*", markers = "platform_machine != 'aarch64'" } # broken on arm64
|
||||
pywinctl = "*"
|
||||
pyprof2calltree = "*"
|
||||
rerun-sdk = "*"
|
||||
tabulate = "*"
|
||||
types-requests = "*"
|
||||
types-tabulate = "*"
|
||||
|
||||
# this is only pinned since 5.15.11 is broken
|
||||
pyqt5 = { version = "==5.15.2", markers = "platform_machine == 'x86_64'" } # no aarch64 wheels for macOS/linux
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
# https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml
|
||||
[tool.ruff]
|
||||
indent-width = 2
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ blacklist = [
|
|||
"^common/tests/",
|
||||
|
||||
# particularly large text files
|
||||
"poetry.lock",
|
||||
"uv.lock",
|
||||
"third_party/catch2",
|
||||
"selfdrive/car/tests/test_models.*",
|
||||
|
||||
|
|
|
|||
|
|
@ -26,16 +26,23 @@ git clone --recurse-submodules https://github.com/commaai/openpilot.git
|
|||
|
||||
``` bash
|
||||
cd openpilot
|
||||
git lfs pull
|
||||
tools/ubuntu_setup.sh
|
||||
```
|
||||
|
||||
Activate a shell with the Python dependencies installed:
|
||||
**3. Git LFS**
|
||||
|
||||
``` bash
|
||||
poetry shell
|
||||
git lfs pull
|
||||
```
|
||||
|
||||
**3. Build openpilot**
|
||||
**4. Activate a python shell**
|
||||
|
||||
Activate a shell with the Python dependencies installed:
|
||||
``` bash
|
||||
source .venv/bin/activate
|
||||
```
|
||||
|
||||
**5. Build openpilot**
|
||||
|
||||
``` bash
|
||||
scons -u -j$(nproc)
|
||||
|
|
|
|||
|
|
@ -8,22 +8,17 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
|
|||
ROOT=$DIR/../
|
||||
cd $ROOT
|
||||
|
||||
RC_FILE="${HOME}/.$(basename ${SHELL})rc"
|
||||
if [ "$(uname)" == "Darwin" ] && [ $SHELL == "/bin/bash" ]; then
|
||||
RC_FILE="$HOME/.bash_profile"
|
||||
fi
|
||||
|
||||
if ! command -v "poetry" > /dev/null 2>&1; then
|
||||
echo "installing poetry..."
|
||||
curl -sSL https://install.python-poetry.org | python3 -
|
||||
POETRY_BIN='$HOME/.local/bin'
|
||||
ADD_PATH_CMD="export PATH=\"$POETRY_BIN:\$PATH\""
|
||||
if ! command -v "uv" > /dev/null 2>&1; then
|
||||
echo "installing uv..."
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
UV_BIN='$HOME/.cargo/env'
|
||||
ADD_PATH_CMD=". \"$UV_BIN\""
|
||||
eval $ADD_PATH_CMD
|
||||
printf "\n#poetry path\n$ADD_PATH_CMD\n" >> $RC_FILE
|
||||
fi
|
||||
|
||||
poetry config virtualenvs.prefer-active-python true --local
|
||||
poetry config virtualenvs.in-project true --local
|
||||
echo "installing python packages..."
|
||||
uv sync --all-extras
|
||||
source .venv/bin/activate
|
||||
|
||||
echo "PYTHONPATH=${PWD}" > $ROOT/.env
|
||||
if [[ "$(uname)" == 'Darwin' ]]; then
|
||||
|
|
@ -32,15 +27,8 @@ if [[ "$(uname)" == 'Darwin' ]]; then
|
|||
echo "export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES" >> $ROOT/.env
|
||||
fi
|
||||
|
||||
poetry self add poetry-dotenv-plugin@^0.1.0
|
||||
|
||||
echo "installing python packages..."
|
||||
poetry install --no-cache --no-root
|
||||
|
||||
[ -n "$POETRY_VIRTUALENVS_CREATE" ] && RUN="" || RUN="poetry run"
|
||||
|
||||
if [ "$(uname)" != "Darwin" ] && [ -e "$ROOT/.git" ]; then
|
||||
echo "pre-commit hooks install..."
|
||||
$RUN pre-commit install
|
||||
$RUN git submodule foreach pre-commit install
|
||||
pre-commit install
|
||||
git submodule foreach pre-commit install
|
||||
fi
|
||||
|
|
|
|||
Loading…
Reference in New Issue