mawei/onepilot
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

CI: replace docker with op setup (#37282)

Browse Source
This commit is contained in:
Adeeb Shihadeh
2026-02-20 15:48:09 -08:00
committed by GitHub
parent 5d54743d8b
commit 806655b052
8 changed files with 49 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.github/workflows/auto-cache/action.yaml vendored
View File

@@ -52,7 +52,4 @@ runs:
# make the directory manually in case we didn't get a hit, so it doesn't fail on future steps
- id: scons-cache-setup
shell: bash
run: |
mkdir -p ${{ inputs.path }}
sudo chmod -R 777 ${{ inputs.path }}
sudo chown -R $USER ${{ inputs.path }}
run: mkdir -p ${{ inputs.path }}

6
.github/workflows/badges.yaml vendored
View File

@@ -5,9 +5,7 @@ on:
workflow_dispatch:
env:
BASE_IMAGE: openpilot-base
DOCKER_REGISTRY: ghcr.io/commaai
RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/bash -c
PYTHONPATH: ${{ github.workspace }}
jobs:
badges:
@@ -23,7 +21,7 @@ jobs:
- uses: ./.github/workflows/setup-with-retry
- name: Push badges
run: |
${{ env.RUN }} "python3 selfdrive/ui/translations/create_badges.py"
python3 selfdrive/ui/translations/create_badges.py
rm .gitattributes

10
.github/workflows/compile-openpilot/action.yaml vendored
View File

@@ -6,16 +6,16 @@ runs:
- shell: bash
name: Build openpilot with all flags
run: |
${{ env.RUN }} "scons -j$(nproc)"
${{ env.RUN }} "release/check-dirty.sh"
scons -j$(nproc)
release/check-dirty.sh
- shell: bash
name: Cleanup scons cache and rebuild
run: |
${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \
scons -j$(nproc) --cache-populate"
rm -rf /tmp/scons_cache/*
scons -j$(nproc) --cache-populate
- name: Save scons cache
uses: actions/cache/save@v4
if: github.ref == 'refs/heads/master'
with:
path: .ci_cache/scons_cache
path: /tmp/scons_cache
key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}

17
.github/workflows/repo-maintenance.yaml vendored
View File

@@ -6,9 +6,7 @@ on:
workflow_dispatch:
env:
BASE_IMAGE: openpilot-base
BUILD: selfdrive/test/docker_build.sh base
RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
PYTHONPATH: ${{ github.workspace }}
jobs:
update_translations:
@@ -20,8 +18,7 @@ jobs:
submodules: true
- uses: ./.github/workflows/setup-with-retry
- name: Update translations
run: |
${{ env.RUN }} "python3 selfdrive/ui/update_translations.py --vanish"
run: python3 selfdrive/ui/update_translations.py --vanish
- name: Create Pull Request
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0
with:
@@ -37,18 +34,14 @@ jobs:
package_updates:
name: package_updates
runs-on: ubuntu-latest
container:
image: ghcr.io/commaai/openpilot-base:latest
if: github.repository == 'commaai/openpilot'
steps:
- uses: actions/checkout@v6
with:
submodules: true
- uses: ./.github/workflows/setup-with-retry
- name: uv lock
run: |
python3 -m ensurepip --upgrade
pip3 install uv
uv lock --upgrade
run: uv lock --upgrade
- name: uv pip tree
id: pip_tree
run: |
@@ -57,12 +50,10 @@ jobs:
echo 'EOF' >> $GITHUB_OUTPUT
- name: bump submodules
run: |
git config --global --add safe.directory '*'
git submodule update --remote
git add .
- name: update car docs
run: |
export PYTHONPATH="$PWD"
scons -j$(nproc) --minimal opendbc_repo
python selfdrive/car/docs.py
git add docs/CARS.md

4
.github/workflows/setup-with-retry/action.yaml vendored
View File

@@ -1,10 +1,6 @@
name: 'openpilot env setup, with retry on failure'
inputs:
docker_hub_pat:
description: 'Auth token for Docker Hub, required for BuildJet jobs'
required: false
default: ''
sleep_time:
description: 'Time to sleep between retries'
required: false

20
.github/workflows/setup/action.yaml vendored
View File

@@ -34,23 +34,19 @@ runs:
- id: date
shell: bash
run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
- shell: bash
run: echo "$CACHE_COMMIT_DATE"
- id: scons-cache
uses: ./.github/workflows/auto-cache
with:
path: .ci_cache/scons_cache
path: /tmp/scons_cache
key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
restore-keys: |
scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}
scons-${{ runner.arch }}
# as suggested here: https://github.com/moby/moby/issues/32816#issuecomment-910030001
- id: normalize-file-permissions
shell: bash
name: Normalize file permissions to ensure a consistent docker build cache
run: |
find . -type f -executable -not -perm 755 -exec chmod 755 {} \;
find . -type f -not -executable -not -perm 644 -exec chmod 644 {} \;
# build our docker image
- shell: bash
run: eval ${{ env.BUILD }}
name: Run setup
run: ./tools/op.sh setup
- shell: bash
name: Setup cache dirs
run: |
mkdir -p /tmp/comma_download_cache
echo "$GITHUB_WORKSPACE/.venv/bin" >> $GITHUB_PATH

78
.github/workflows/tests.yaml vendored
View File

@@ -18,13 +18,8 @@ concurrency:
cancel-in-progress: true
env:
PYTHONWARNINGS: error
BASE_IMAGE: openpilot-base
DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
BUILD: selfdrive/test/docker_build.sh base
RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
CI: 1
PYTHONPATH: ${{ github.workspace }}
PYTEST: pytest --continue-on-collection-errors --durations=0 -n logical
jobs:
@@ -38,6 +33,7 @@ jobs:
|| fromJSON('["ubuntu-24.04"]') }}
env:
STRIPPED_DIR: /tmp/releasepilot
PYTHONPATH: /tmp/releasepilot
steps:
- uses: actions/checkout@v6
with:
@@ -53,15 +49,13 @@ jobs:
run: TARGET_DIR=$STRIPPED_DIR release/build_stripped.sh
- uses: ./.github/workflows/setup-with-retry
- name: Build openpilot and run checks
timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
run: |
cd $STRIPPED_DIR
${{ env.RUN }} "python3 system/manager/build.py"
timeout-minutes: 30
working-directory: ${{ env.STRIPPED_DIR }}
run: python3 system/manager/build.py
- name: Run tests
timeout-minutes: 1
run: |
cd $STRIPPED_DIR
${{ env.RUN }} "release/check-dirty.sh"
working-directory: ${{ env.STRIPPED_DIR }}
run: release/check-dirty.sh
- name: Check submodules
if: github.repository == 'commaai/openpilot'
timeout-minutes: 3
@@ -78,11 +72,6 @@ jobs:
- uses: actions/checkout@v6
with:
submodules: true
- name: Setup docker push
if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
run: |
echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
$DOCKER_LOGIN
- uses: ./.github/workflows/setup-with-retry
- uses: ./.github/workflows/compile-openpilot
timeout-minutes: 30
@@ -106,7 +95,6 @@ jobs:
- name: Install dependencies
run: ./tools/mac_setup.sh
env:
PYTHONWARNINGS: default # package install has DeprecationWarnings
HOMEBREW_DISPLAY_INSTALL_TIMES: 1
- run: git lfs pull
- name: Getting scons cache
@@ -128,8 +116,6 @@ jobs:
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
&& fromJSON('["namespace-profile-amd64-8x16", "namespace-experiments:docker.builds.local-cache=separate"]')
|| fromJSON('["ubuntu-24.04"]') }}
env:
PYTHONWARNINGS: default
steps:
- uses: actions/checkout@v6
with:
@@ -137,7 +123,7 @@ jobs:
- uses: ./.github/workflows/setup-with-retry
- name: Static analysis
timeout-minutes: 1
run: ${{ env.RUN }} "scripts/lint/lint.sh"
run: scripts/lint/lint.sh
unit_tests:
name: unit tests
@@ -154,15 +140,14 @@ jobs:
- uses: ./.github/workflows/setup-with-retry
id: setup-step
- name: Build openpilot
run: ${{ env.RUN }} "scons -j$(nproc)"
run: scons -j$(nproc)
- name: Run unit tests
timeout-minutes: ${{ contains(runner.name, 'nsc') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 20 }}
run: |
${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
# Pre-compile Python bytecode so each pytest worker doesn't need to
$PYTEST --collect-only -m 'not slow' -qq && \
MAX_EXAMPLES=1 $PYTEST -m 'not slow' && \
chmod -R 777 /tmp/comma_download_cache"
source selfdrive/test/setup_xvfb.sh
# Pre-compile Python bytecode so each pytest worker doesn't need to
$PYTEST --collect-only -m 'not slow' -qq
MAX_EXAMPLES=1 $PYTEST -m 'not slow'
process_replay:
name: process replay
@@ -182,17 +167,14 @@ jobs:
id: dependency-cache
uses: actions/cache@v5
with:
path: .ci_cache/comma_download_cache
path: /tmp/comma_download_cache
key: proc-replay-${{ hashFiles('selfdrive/test/process_replay/test_processes.py') }}
- name: Build openpilot
run: |
${{ env.RUN }} "scons -j$(nproc)"
run: scons -j$(nproc)
- name: Run replay
timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && ((steps.setup-step.outputs.duration < 18) && 1 || 2) || 20 }}
continue-on-error: ${{ github.ref == 'refs/heads/master' }}
run: |
${{ env.RUN }} "selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
chmod -R 777 /tmp/comma_download_cache"
run: selfdrive/test/process_replay/test_processes.py -j$(nproc)
- name: Print diff
id: print-diff
if: always()
@@ -226,9 +208,9 @@ jobs:
- name: Run regen
if: false
timeout-minutes: 4
run: |
${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \
chmod -R 777 /tmp/comma_download_cache"
env:
ONNXCPU: 1
run: $PYTEST selfdrive/test/process_replay/test_regen.py
simulator_driving:
name: simulator driving
@@ -246,14 +228,13 @@ jobs:
- uses: ./.github/workflows/setup-with-retry
id: setup-step
- name: Build openpilot
run: |
${{ env.RUN }} "scons -j$(nproc)"
run: scons -j$(nproc)
- name: Driving test
timeout-minutes: ${{ (steps.setup-step.outputs.duration < 18) && 1 || 2 }}
run: |
${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
source selfdrive/test/setup_vsound.sh && \
CI=1 pytest -s tools/sim/tests/test_metadrive_bridge.py"
source selfdrive/test/setup_xvfb.sh
source selfdrive/test/setup_vsound.sh
pytest -s tools/sim/tests/test_metadrive_bridge.py
create_ui_report:
name: Create UI Report
@@ -269,13 +250,12 @@ jobs:
submodules: true
- uses: ./.github/workflows/setup-with-retry
- name: Build openpilot
run: ${{ env.RUN }} "scons -j$(nproc)"
run: scons -j$(nproc)
- name: Create UI Report
run: >
${{ env.RUN }} "PYTHONWARNINGS=ignore &&
source selfdrive/test/setup_xvfb.sh &&
python3 selfdrive/ui/tests/diff/replay.py &&
python3 selfdrive/ui/tests/diff/replay.py --big"
run: |
source selfdrive/test/setup_xvfb.sh
python3 selfdrive/ui/tests/diff/replay.py
python3 selfdrive/ui/tests/diff/replay.py --big
- name: Upload UI Report
uses: actions/upload-artifact@v6
with:

65
system/webrtc/tests/test_webrtcd.py
View File

@@ -1,65 +0,0 @@
import pytest
import asyncio
import json
# for aiortc and its dependencies
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
warnings.filterwarnings("ignore", category=RuntimeWarning) # TODO: remove this when google-crc32c publish a python3.12 wheel
from openpilot.system.webrtc.webrtcd import get_stream
import aiortc
from teleoprtc import WebRTCOfferBuilder
from openpilot.common.parameterized import parameterized_class
@parameterized_class(("in_services", "out_services"), [
(["testJoystick"], ["carState"]),
([], ["carState"]),
(["testJoystick"], []),
([], []),
])
@pytest.mark.asyncio
class TestWebrtcdProc:
async def assertCompletesWithTimeout(self, awaitable, timeout=1):
try:
async with asyncio.timeout(timeout):
await awaitable
except TimeoutError:
pytest.fail("Timeout while waiting for awaitable to complete")
async def test_webrtcd(self, mocker):
mock_request = mocker.MagicMock()
async def connect(offer):
body = {'sdp': offer.sdp, 'cameras': offer.video, 'bridge_services_in': self.in_services, 'bridge_services_out': self.out_services}
mock_request.json.side_effect = mocker.AsyncMock(return_value=body)
response = await get_stream(mock_request)
response_json = json.loads(response.text)
return aiortc.RTCSessionDescription(**response_json)
builder = WebRTCOfferBuilder(connect)
builder.offer_to_receive_video_stream("road")
builder.offer_to_receive_audio_stream()
if len(self.in_services) > 0 or len(self.out_services) > 0:
builder.add_messaging()
stream = builder.stream()
await self.assertCompletesWithTimeout(stream.start())
await self.assertCompletesWithTimeout(stream.wait_for_connection())
assert stream.has_incoming_video_track("road")
assert stream.has_incoming_audio_track()
assert stream.has_messaging_channel() == (len(self.in_services) > 0 or len(self.out_services) > 0)
video_track, audio_track = stream.get_incoming_video_track("road"), stream.get_incoming_audio_track()
await self.assertCompletesWithTimeout(video_track.recv())
await self.assertCompletesWithTimeout(audio_track.recv())
await self.assertCompletesWithTimeout(stream.stop())
# cleanup, very implementation specific, test may break if it changes
assert mock_request.app["streams"].__setitem__.called, "Implementation changed, please update this test"
_, session = mock_request.app["streams"].__setitem__.call_args.args
await self.assertCompletesWithTimeout(session.post_run_cleanup())