simplify setup (#37358)

* simplify setup

* lil more

* simplify dockedr

* just run setup there:

* don't need that junk

* lil more

.github/workflows/prebuilt.yaml

@@ -6,7 +6,7 @@ on:
 
 env:
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-  BUILD: selfdrive/test/docker_build.sh prebuilt
+  BUILD: selfdrive/test/docker_build.sh
 
 jobs:
   build_prebuilt:

.github/workflows/release.yaml

@@ -7,20 +7,12 @@ on:
 jobs:
   build_masterci:
     name: build master-ci
-    env:
-      ImageOS: ubuntu24
-    container:
-      image: ghcr.io/commaai/openpilot-base:latest
     runs-on: ubuntu-latest
     if: github.repository == 'commaai/openpilot'
     permissions:
       checks: read
       contents: write
     steps:
-    - name: Install wait-on-check-action dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y libyaml-dev
     - name: Wait for green check mark
       if: ${{ github.event_name == 'schedule' }}
       uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc
@@ -34,9 +26,6 @@ jobs:
       with:
         submodules: true
         fetch-depth: 0
-    - name: Pull LFS
-      run: |
-        git config --global --add safe.directory '*'
-        git lfs pull
+    - uses: ./.github/workflows/setup-with-retry
     - name: Push master-ci
       run: BRANCH=__nightly release/build_stripped.sh

Dockerfile.openpilot

@@ -1,14 +1,38 @@
-FROM ghcr.io/commaai/openpilot-base:latest
+FROM ubuntu:24.04
 
 ENV PYTHONUNBUFFERED=1
 
-ENV OPENPILOT_PATH=/home/batman/openpilot
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends sudo tzdata locales && \
+    rm -rf /var/lib/apt/lists/*
 
+RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
+ENV LANG=en_US.UTF-8
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8
+
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
+
+ARG USER=batman
+ARG USER_UID=1001
+RUN useradd -m -s /bin/bash -u $USER_UID $USER
+RUN usermod -aG sudo $USER
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+USER $USER
+
+ENV OPENPILOT_PATH=/home/$USER/openpilot
 RUN mkdir -p ${OPENPILOT_PATH}
 WORKDIR ${OPENPILOT_PATH}
 
-COPY . ${OPENPILOT_PATH}/
+COPY --chown=$USER . ${OPENPILOT_PATH}/
 
-ENV UV_BIN="/home/batman/.local/bin/"
-ENV PATH="$UV_BIN:$PATH"
-RUN UV_PROJECT_ENVIRONMENT=$VIRTUAL_ENV uv run scons --cache-readonly -j$(nproc)
+ENV UV_BIN="/home/$USER/.local/bin/"
+ENV VIRTUAL_ENV=${OPENPILOT_PATH}/.venv
+ENV PATH="$UV_BIN:$VIRTUAL_ENV/bin:$PATH"
+RUN tools/setup_dependencies.sh && \
+    sudo rm -rf /var/lib/apt/lists/*
+
+USER root
+RUN git config --global --add safe.directory '*'

Dockerfile.openpilot_base

@@ -1,42 +0,0 @@
-FROM ubuntu:24.04
-
-ENV PYTHONUNBUFFERED=1
-
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot python3-tk python3-dev && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
-ENV LANG=en_US.UTF-8
-ENV LANGUAGE=en_US:en
-ENV LC_ALL=en_US.UTF-8
-
-COPY tools/install_ubuntu_dependencies.sh /tmp/tools/
-RUN /tmp/tools/install_ubuntu_dependencies.sh && \
-    rm -rf /var/lib/apt/lists/* /tmp/*
-
-ENV NVIDIA_VISIBLE_DEVICES=all
-ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
-ENV QTWEBENGINE_DISABLE_SANDBOX=1
-
-RUN dbus-uuidgen > /etc/machine-id
-
-ARG USER=batman
-ARG USER_UID=1001
-RUN useradd -m -s /bin/bash -u $USER_UID $USER
-RUN usermod -aG sudo $USER
-RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-USER $USER
-
-COPY --chown=$USER pyproject.toml uv.lock /home/$USER
-COPY --chown=$USER tools/install_python_dependencies.sh /home/$USER/tools/
-
-ENV VIRTUAL_ENV=/home/$USER/.venv
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-RUN cd /home/$USER && \
-    tools/install_python_dependencies.sh && \
-    rm -rf tools/ pyproject.toml uv.lock .cache
-
-USER root
-RUN sudo git config --global --add safe.directory /tmp/openpilot

selfdrive/test/docker_build.sh

@@ -1,12 +1,14 @@
 #!/usr/bin/env bash
 set -e
 
-# To build sim and docs, you can run the following to mount the scons cache to the same place as in CI:
-# mkdir -p .ci_cache/scons_cache
-# sudo mount --bind /tmp/scons_cache/ .ci_cache/scons_cache
-
 SCRIPT_DIR=$(dirname "$0")
 OPENPILOT_DIR=$SCRIPT_DIR/../../
+
+DOCKER_IMAGE=openpilot
+DOCKER_FILE=Dockerfile.openpilot
+DOCKER_REGISTRY=ghcr.io/commaai
+COMMIT_SHA=$(git rev-parse HEAD)
+
 if [ -n "$TARGET_ARCHITECTURE" ]; then
   PLATFORM="linux/$TARGET_ARCHITECTURE"
   TAG_SUFFIX="-$TARGET_ARCHITECTURE"
@@ -15,9 +17,11 @@ else
   TAG_SUFFIX=""
 fi
 
-source $SCRIPT_DIR/docker_common.sh $1 "$TAG_SUFFIX"
+LOCAL_TAG=$DOCKER_IMAGE$TAG_SUFFIX
+REMOTE_TAG=$DOCKER_REGISTRY/$LOCAL_TAG
+REMOTE_SHA_TAG=$DOCKER_REGISTRY/$LOCAL_TAG:$COMMIT_SHA
 
-DOCKER_BUILDKIT=1 docker buildx build --provenance false --pull --platform $PLATFORM --load --cache-to type=inline --cache-from type=registry,ref=$REMOTE_TAG -t $DOCKER_IMAGE:latest -t $REMOTE_TAG -t $LOCAL_TAG -f $OPENPILOT_DIR/$DOCKER_FILE $OPENPILOT_DIR
+DOCKER_BUILDKIT=1 docker buildx build --provenance false --pull --platform $PLATFORM --load -t $DOCKER_IMAGE:latest -t $REMOTE_TAG -t $LOCAL_TAG -f $OPENPILOT_DIR/$DOCKER_FILE $OPENPILOT_DIR
 
 if [ -n "$PUSH_IMAGE" ]; then
   docker push $REMOTE_TAG

selfdrive/test/docker_common.sh

@@ -1,18 +0,0 @@
-if [ "$1" = "base" ]; then
-  export DOCKER_IMAGE=openpilot-base
-  export DOCKER_FILE=Dockerfile.openpilot_base
-elif [ "$1" = "prebuilt" ]; then
-  export DOCKER_IMAGE=openpilot-prebuilt
-  export DOCKER_FILE=Dockerfile.openpilot
-else
-  echo "Invalid docker build image: '$1'"
-  exit 1
-fi
-
-export DOCKER_REGISTRY=ghcr.io/commaai
-export COMMIT_SHA=$(git rev-parse HEAD)
-
-TAG_SUFFIX=$2
-LOCAL_TAG=$DOCKER_IMAGE$TAG_SUFFIX
-REMOTE_TAG=$DOCKER_REGISTRY/$LOCAL_TAG
-REMOTE_SHA_TAG=$DOCKER_REGISTRY/$LOCAL_TAG:$COMMIT_SHA

tools/install_python_dependencies.sh

@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-# Increase the pip timeout to handle TimeoutError
-export PIP_DEFAULT_TIMEOUT=200
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-ROOT="$DIR"/../
-cd "$ROOT"
-
-if ! command -v "uv" > /dev/null 2>&1; then
-  echo "installing uv..."
-  curl -LsSf --retry 5 --retry-delay 5 --retry-all-errors https://astral.sh/uv/install.sh | sh
-  UV_BIN="$HOME/.local/bin"
-  PATH="$UV_BIN:$PATH"
-fi
-
-echo "updating uv..."
-# ok to fail, can also fail due to installing with brew
-uv self update || true
-
-echo "installing python packages..."
-uv sync --frozen --all-extras
-source .venv/bin/activate
-
-if [[ "$(uname)" == 'Darwin' ]]; then
-  touch "$ROOT"/.env
-  echo "export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES" >> "$ROOT"/.env
-fi

tools/install_ubuntu_dependencies.sh

@@ -1,99 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-SUDO=""
-
-# Use sudo if not root
-if [[ ! $(id -u) -eq 0 ]]; then
-  if [[ -z $(which sudo) ]]; then
-    echo "Please install sudo or run as root"
-    exit 1
-  fi
-  SUDO="sudo"
-fi
-
-# Check if stdin is open
-if [ -t 0 ]; then
-  INTERACTIVE=1
-fi
-
-# Install common packages
-function install_ubuntu_common_requirements() {
-  $SUDO apt-get update
-
-  # normal stuff, mostly for the bare docker image
-  $SUDO apt-get install -y --no-install-recommends \
-    ca-certificates \
-    build-essential \
-    curl \
-    libssl-dev \
-    libcurl4-openssl-dev \
-    locales \
-    git \
-    xvfb
-
-  $SUDO apt-get install -y --no-install-recommends \
-    libgles2-mesa-dev \
-    libjpeg-dev \
-    libncurses5-dev \
-    libzstd-dev \
-    gettext
-}
-
-# Install Ubuntu 24.04 LTS packages
-function install_ubuntu_lts_latest_requirements() {
-  install_ubuntu_common_requirements
-
-  $SUDO apt-get install -y --no-install-recommends \
-    python3-dev \
-    python3-venv
-}
-
-# Detect OS using /etc/os-release file
-if [ -f "/etc/os-release" ]; then
-  source /etc/os-release
-  case "$VERSION_CODENAME" in
-    "jammy" | "kinetic" | "noble")
-      install_ubuntu_lts_latest_requirements
-      ;;
-    *)
-      echo "$ID $VERSION_ID is unsupported. This setup script is written for Ubuntu 24.04."
-      read -p "Would you like to attempt installation anyway? " -n 1 -r
-      echo ""
-      if [[ ! $REPLY =~ ^[Yy]$ ]]; then
-        exit 1
-      fi
-      install_ubuntu_lts_latest_requirements
-  esac
-
-  if [[ -d "/etc/udev/rules.d/" ]]; then
-    # Setup jungle udev rules
-    $SUDO tee /etc/udev/rules.d/12-panda_jungle.rules > /dev/null <<EOF
-SUBSYSTEM=="usb", ATTRS{idVendor}=="3801", ATTRS{idProduct}=="ddcf", MODE="0666"
-SUBSYSTEM=="usb", ATTRS{idVendor}=="3801", ATTRS{idProduct}=="ddef", MODE="0666"
-SUBSYSTEM=="usb", ATTRS{idVendor}=="bbaa", ATTRS{idProduct}=="ddcf", MODE="0666"
-SUBSYSTEM=="usb", ATTRS{idVendor}=="bbaa", ATTRS{idProduct}=="ddef", MODE="0666"
-
-EOF
-
-    # Setup panda udev rules
-    $SUDO tee /etc/udev/rules.d/11-panda.rules > /dev/null <<EOF
-SUBSYSTEM=="usb", ATTRS{idVendor}=="0483", ATTRS{idProduct}=="df11", MODE="0666"
-SUBSYSTEM=="usb", ATTRS{idVendor}=="3801", ATTRS{idProduct}=="ddcc", MODE="0666"
-SUBSYSTEM=="usb", ATTRS{idVendor}=="3801", ATTRS{idProduct}=="ddee", MODE="0666"
-SUBSYSTEM=="usb", ATTRS{idVendor}=="bbaa", ATTRS{idProduct}=="ddcc", MODE="0666"
-SUBSYSTEM=="usb", ATTRS{idVendor}=="bbaa", ATTRS{idProduct}=="ddee", MODE="0666"
-EOF
-
-    # Setup adb udev rules
-    $SUDO tee /etc/udev/rules.d/50-comma-adb.rules > /dev/null <<EOF
-SUBSYSTEM=="usb", ATTR{idVendor}=="04d8", ATTR{idProduct}=="1234", ENV{adb_user}="yes"
-EOF
-
-    $SUDO udevadm control --reload-rules && $SUDO udevadm trigger || true
-  fi
-
-else
-  echo "No /etc/os-release in the system. Make sure you're running on Ubuntu, or similar."
-  exit 1
-fi

tools/mac_setup.sh

@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-ROOT="$(cd $DIR/../ && pwd)"
-
-if [[ $SHELL == "/bin/zsh" ]]; then
-  RC_FILE="$HOME/.zshrc"
-elif [[ $SHELL == "/bin/bash" ]]; then
-  RC_FILE="$HOME/.bash_profile"
-fi
-
-# install python dependencies
-$DIR/install_python_dependencies.sh
-echo "[ ] installed python dependencies t=$SECONDS"
-
-echo
-echo "----   OPENPILOT SETUP DONE   ----"
-echo "Open a new shell or configure your active shell env by running:"
-echo "source $RC_FILE"

tools/op.sh

@@ -216,11 +216,7 @@ function op_setup() {
 
   echo "Installing dependencies..."
   st="$(date +%s)"
-  if [[ "$OSTYPE" == "linux-gnu"* ]]; then
-    SETUP_SCRIPT="tools/ubuntu_setup.sh"
-  elif [[ "$OSTYPE" == "darwin"* ]]; then
-    SETUP_SCRIPT="tools/mac_setup.sh"
-  fi
+  SETUP_SCRIPT="tools/setup_dependencies.sh"
   if ! $OPENPILOT_ROOT/$SETUP_SCRIPT; then
     echo -e " ↳ [${RED}✗${NC}] Dependencies installation failed!"
     loge "ERROR_DEPENDENCIES_INSTALLATION"

tools/setup_dependencies.sh

@@ -0,0 +1,137 @@
+#!/usr/bin/env bash
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+ROOT="$(cd "$DIR/../" && pwd)"
+
+function install_ubuntu_deps() {
+  SUDO=""
+
+  if [[ ! $(id -u) -eq 0 ]]; then
+    if [[ -z $(which sudo) ]]; then
+      echo "Please install sudo or run as root"
+      exit 1
+    fi
+    SUDO="sudo"
+  fi
+
+  # Detect OS using /etc/os-release file
+  if [ -f "/etc/os-release" ]; then
+    source /etc/os-release
+    case "$VERSION_CODENAME" in
+      "jammy" | "kinetic" | "noble")
+        ;;
+      *)
+        echo "$ID $VERSION_ID is unsupported. This setup script is written for Ubuntu 24.04."
+        read -p "Would you like to attempt installation anyway? " -n 1 -r
+        echo ""
+        if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+          exit 1
+        fi
+        ;;
+    esac
+  else
+    echo "No /etc/os-release in the system. Make sure you're running on Ubuntu, or similar."
+    exit 1
+  fi
+
+  $SUDO apt-get update
+
+  # normal stuff, mostly for the bare docker image
+  $SUDO apt-get install -y --no-install-recommends \
+    ca-certificates \
+    build-essential \
+    curl \
+    libssl-dev \
+    libcurl4-openssl-dev \
+    locales \
+    git \
+    xvfb
+
+  $SUDO apt-get install -y --no-install-recommends \
+    python3-dev \
+    libgles2-mesa-dev \
+    libjpeg-dev \
+    libncurses5-dev \
+    libzstd-dev \
+    gettext
+
+  if [[ -d "/etc/udev/rules.d/" ]]; then
+    # Setup jungle udev rules
+    $SUDO tee /etc/udev/rules.d/12-panda_jungle.rules > /dev/null <<EOF
+SUBSYSTEM=="usb", ATTRS{idVendor}=="3801", ATTRS{idProduct}=="ddcf", MODE="0666"
+SUBSYSTEM=="usb", ATTRS{idVendor}=="3801", ATTRS{idProduct}=="ddef", MODE="0666"
+SUBSYSTEM=="usb", ATTRS{idVendor}=="bbaa", ATTRS{idProduct}=="ddcf", MODE="0666"
+SUBSYSTEM=="usb", ATTRS{idVendor}=="bbaa", ATTRS{idProduct}=="ddef", MODE="0666"
+
+EOF
+
+    # Setup panda udev rules
+    $SUDO tee /etc/udev/rules.d/11-panda.rules > /dev/null <<EOF
+SUBSYSTEM=="usb", ATTRS{idVendor}=="0483", ATTRS{idProduct}=="df11", MODE="0666"
+SUBSYSTEM=="usb", ATTRS{idVendor}=="3801", ATTRS{idProduct}=="ddcc", MODE="0666"
+SUBSYSTEM=="usb", ATTRS{idVendor}=="3801", ATTRS{idProduct}=="ddee", MODE="0666"
+SUBSYSTEM=="usb", ATTRS{idVendor}=="bbaa", ATTRS{idProduct}=="ddcc", MODE="0666"
+SUBSYSTEM=="usb", ATTRS{idVendor}=="bbaa", ATTRS{idProduct}=="ddee", MODE="0666"
+EOF
+
+    # Setup adb udev rules
+    $SUDO tee /etc/udev/rules.d/50-comma-adb.rules > /dev/null <<EOF
+SUBSYSTEM=="usb", ATTR{idVendor}=="04d8", ATTR{idProduct}=="1234", ENV{adb_user}="yes"
+EOF
+
+    $SUDO udevadm control --reload-rules && $SUDO udevadm trigger || true
+  fi
+}
+
+function install_python_deps() {
+  # Increase the pip timeout to handle TimeoutError
+  export PIP_DEFAULT_TIMEOUT=200
+
+  cd "$ROOT"
+
+  if ! command -v "uv" > /dev/null 2>&1; then
+    echo "installing uv..."
+    curl -LsSf --retry 5 --retry-delay 5 --retry-all-errors https://astral.sh/uv/install.sh | sh
+    UV_BIN="$HOME/.local/bin"
+    PATH="$UV_BIN:$PATH"
+  fi
+
+  echo "updating uv..."
+  # ok to fail, can also fail due to installing with brew
+  uv self update || true
+
+  echo "installing python packages..."
+  uv sync --frozen --all-extras
+  source .venv/bin/activate
+
+  if [[ "$(uname)" == 'Darwin' ]]; then
+    touch "$ROOT"/.env
+    echo "export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES" >> "$ROOT"/.env
+  fi
+}
+
+# --- Main ---
+
+if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+  install_ubuntu_deps
+  echo "[ ] installed system dependencies t=$SECONDS"
+elif [[ "$OSTYPE" == "darwin"* ]]; then
+  if [[ $SHELL == "/bin/zsh" ]]; then
+    RC_FILE="$HOME/.zshrc"
+  elif [[ $SHELL == "/bin/bash" ]]; then
+    RC_FILE="$HOME/.bash_profile"
+  fi
+fi
+
+if [ -f "$ROOT/pyproject.toml" ]; then
+  install_python_deps
+  echo "[ ] installed python dependencies t=$SECONDS"
+fi
+
+if [[ "$OSTYPE" == "darwin"* ]] && [[ -n "${RC_FILE:-}" ]]; then
+  echo
+  echo "----   OPENPILOT SETUP DONE   ----"
+  echo "Open a new shell or configure your active shell env by running:"
+  echo "source $RC_FILE"
+fi

tools/ubuntu_setup.sh

@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-
-# NOTE: this is used in a docker build, so do not run any scripts here.
-
-"$DIR"/install_ubuntu_dependencies.sh
-"$DIR"/install_python_dependencies.sh