diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ee2e144b..82bc2345 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -80,4 +80,4 @@ jobs: - name: tests/misra/install.sh run: ${{ env.RUN }} "cd tests/misra && ./install.sh" - name: MISRA mutation tests - run: ${{ env.RUN }} "cd tests/misra && pytest -n2 test_mutation.py" + run: ${{ env.RUN }} "cd tests/misra && pytest test_mutation.py" diff --git a/Jenkinsfile b/Jenkinsfile index 22786b1d..2f5341c0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -109,7 +109,7 @@ pipeline { ["build", "scons -j4"], ["flash", "cd scripts/ && ./reflash_internal_panda.py"], ["flash jungle", "cd board/jungle && ./flash.py --all"], - ["test", "cd tests/hitl && HW_TYPES=10 pytest -n0 --durations=0 2*.py [5-9]*.py"], + ["test", "cd tests/hitl && HW_TYPES=10 pytest --durations=0 2*.py [5-9]*.py"], ]) } } @@ -121,7 +121,7 @@ pipeline { ["build", "scons -j4"], ["flash", "cd scripts/ && ./reflash_internal_panda.py"], ["flash jungle", "cd board/jungle && ./flash.py --all"], - ["test", "cd tests/hitl && HW_TYPES=9 pytest -n0 --durations=0 2*.py [5-9]*.py"], + ["test", "cd tests/hitl && HW_TYPES=9 pytest --durations=0 2*.py [5-9]*.py"], ]) } } @@ -133,7 +133,7 @@ pipeline { ["build", "scons -j4"], ["flash", "cd scripts/ && ./reflash_internal_panda.py"], ["flash jungle", "cd board/jungle && ./flash.py --all"], - ["test", "cd tests/hitl && HW_TYPES=6 pytest -n0 --durations=0 [2-9]*.py -k 'not test_send_recv'"], + ["test", "cd tests/hitl && HW_TYPES=6 pytest --durations=0 [2-9]*.py -k 'not test_send_recv'"], ]) } } @@ -141,7 +141,7 @@ pipeline { stage('bootkick tests') { steps { script { - docker_run("test", 10, "pytest -n0 ./tests/som/test_bootkick.py") + docker_run("test", 10, "pytest ./tests/som/test_bootkick.py") } } } diff --git a/pyproject.toml b/pyproject.toml index d3b4f837..8e147c3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "pandacan" version = "0.0.10" description = "Code powering the comma.ai panda" readme = "README.md" -requires-python = ">=3.11,<3.13" +requires-python = ">=3.11,<3.13" # macOS doesn't work with 3.13 due to pycapnp from opendbc license = {text = "MIT"} authors = [{name = "comma.ai"}] classifiers = [ @@ -24,7 +24,6 @@ dev = [ "flaky", "pytest", "pytest-mock", - "pytest-xdist", "pytest-timeout", "pytest-randomly", "ruff", @@ -69,7 +68,7 @@ flake8-implicit-str-concat.allow-multiline=false "pytest.main".msg = "pytest.main requires special handling that is easy to mess up!" [tool.pytest.ini_options] -addopts = "-n0 -Werror --strict-config --strict-markers --durations=10 --ignore-glob='*.sh' --ignore=tests/misra --ignore=tests/som --ignore=tests/hitl" +addopts = "-Werror --strict-config --strict-markers --durations=10 --ignore-glob='*.sh' --ignore=tests/misra --ignore=tests/som --ignore=tests/hitl" python_files = "test_*.py" testpaths = [ "tests/" diff --git a/setup.sh b/setup.sh index 016b1820..41fb8ae2 100755 --- a/setup.sh +++ b/setup.sh @@ -12,11 +12,17 @@ if [[ $PLATFORM == "Darwin" ]]; then brew install --cask gcc-arm-embedded brew install python3 gcc@13 elif [[ $PLATFORM == "Linux" ]]; then + # for AGNOS since we clear the apt lists + if [[ ! -d /"var/lib/apt/" ]]; then + sudo apt update + fi + sudo apt-get install -y --no-install-recommends \ - curl \ + curl ca-certificates \ make g++ git libnewlib-arm-none-eabi \ libusb-1.0-0 \ - gcc-arm-none-eabi python3-pip python3-venv python3-dev + gcc-arm-none-eabi \ + python3-dev python3-pip python3-venv else echo "WARNING: unsupported platform. skipping apt/brew install." fi diff --git a/test.sh b/test.sh index 72879771..6412ce74 100755 --- a/test.sh +++ b/test.sh @@ -17,5 +17,5 @@ mypy python/ # *** test *** -# TODO: make xdist and randomly work -pytest -n0 --randomly-dont-reorganize tests/ +# TODO: make randomly work +pytest --randomly-dont-reorganize tests/ diff --git a/tests/hitl/3_usb_to_can.py b/tests/hitl/3_usb.py similarity index 95% rename from tests/hitl/3_usb_to_can.py rename to tests/hitl/3_usb.py index 638f4337..ba7081a4 100644 --- a/tests/hitl/3_usb_to_can.py +++ b/tests/hitl/3_usb.py @@ -1,10 +1,15 @@ import time +import pytest from flaky import flaky from opendbc.car.structs import CarParams from panda import Panda from panda.tests.hitl.helpers import time_many_sends +pytestmark = [ + pytest.mark.test_panda_types((Panda.HW_TYPE_DOS, Panda.HW_TYPE_RED_PANDA)) +] + def test_can_loopback(p): p.set_safety_mode(CarParams.SafetyModel.allOutput) p.set_can_loopback(True) diff --git a/tests/hitl/4_can_loopback.py b/tests/hitl/4_can_loopback.py index 4c7f926a..63fce9d9 100644 --- a/tests/hitl/4_can_loopback.py +++ b/tests/hitl/4_can_loopback.py @@ -7,7 +7,6 @@ from flaky import flaky from collections import defaultdict from opendbc.car.structs import CarParams -from panda.tests.hitl.conftest import PandaGroup from panda.tests.hitl.helpers import time_many_sends, get_random_can_messages, clear_can_buffers @flaky(max_runs=3, min_passes=1) @@ -90,7 +89,6 @@ def test_latency(p, panda_jungle): @pytest.mark.panda_expect_can_error -@pytest.mark.test_panda_types(PandaGroup.GEN2) def test_gen2_loopback(p, panda_jungle): def test(p_send, p_recv, address=None): for bus in range(4): diff --git a/tests/hitl/9_harness.py b/tests/hitl/9_harness.py index d588fa0a..ddd3516e 100644 --- a/tests/hitl/9_harness.py +++ b/tests/hitl/9_harness.py @@ -4,12 +4,10 @@ import itertools from opendbc.car.structs import CarParams from panda import Panda -from panda.tests.hitl.conftest import PandaGroup # TODO: test relay @pytest.mark.panda_expect_can_error -@pytest.mark.test_panda_types(PandaGroup.GEN2) def test_harness_status(p, panda_jungle): # map from jungle orientations to panda orientations orientation_map = { @@ -61,7 +59,7 @@ def test_harness_status(p, panda_jungle): assert buses[2] == (0 if flipped else 2) # SBU voltages - supply_voltage_mV = 1800 if p.get_type() in [Panda.HW_TYPE_TRES, ] else 3300 + supply_voltage_mV = 1800 if p.get_type() in [Panda.HW_TYPE_TRES, Panda.HW_TYPE_CUATRO] else 3300 if orientation == Panda.HARNESS_STATUS_NC: assert health['sbu1_voltage_mV'] > 0.9 * supply_voltage_mV diff --git a/tests/hitl/conftest.py b/tests/hitl/conftest.py index ed20128f..c831e849 100644 --- a/tests/hitl/conftest.py +++ b/tests/hitl/conftest.py @@ -1,62 +1,32 @@ import os import pytest -import concurrent.futures from panda import Panda, PandaDFU, PandaJungle from panda.tests.hitl.helpers import clear_can_buffers -# needed to get output when using xdist -if "DEBUG" in os.environ: - import sys - sys.stdout = sys.stderr - SPEED_NORMAL = 500 BUS_SPEEDS = [(0, SPEED_NORMAL), (1, SPEED_NORMAL), (2, SPEED_NORMAL)] - -JUNGLE_SERIAL = os.getenv("PANDAS_JUNGLE") +# test options NO_JUNGLE = os.environ.get("NO_JUNGLE", "0") == "1" -PANDAS_EXCLUDE = os.getenv("PANDAS_EXCLUDE", "").strip().split(" ") -HW_TYPES = os.environ.get("HW_TYPES", None) - -PARALLEL = "PARALLEL" in os.environ -NON_PARALLEL = "NON_PARALLEL" in os.environ -if PARALLEL: - NO_JUNGLE = True - -class PandaGroup: - H7 = (Panda.HW_TYPE_RED_PANDA, Panda.HW_TYPE_RED_PANDA_V2, Panda.HW_TYPE_TRES) - GEN2 = (Panda.HW_TYPE_UNO, Panda.HW_TYPE_DOS) + H7 - TESTED = (Panda.HW_TYPE_RED_PANDA, Panda.HW_TYPE_RED_PANDA_V2, Panda.HW_TYPE_UNO) - -if HW_TYPES is not None: - PandaGroup.TESTED = [bytes([int(x), ]) for x in HW_TYPES.strip().split(",")] # type: ignore - # Find all pandas connected -_all_pandas = {} _panda_jungle = None -def init_all_pandas(): +_panda_type = None +_panda_serial = None +def init_devices(): if not NO_JUNGLE: global _panda_jungle - _panda_jungle = PandaJungle(JUNGLE_SERIAL) + _panda_jungle = PandaJungle() _panda_jungle.set_panda_power(True) - for serial in Panda.list(): - if serial not in PANDAS_EXCLUDE: - with Panda(serial=serial, claim=False) as p: - ptype = bytes(p.get_type()) - if ptype in PandaGroup.TESTED: - _all_pandas[serial] = ptype - - # ensure we have all tested panda types - missing_types = set(PandaGroup.TESTED) - set(_all_pandas.values()) - assert len(missing_types) == 0, f"Missing panda types: {missing_types}" - - print(f"{len(_all_pandas)} total pandas") -init_all_pandas() -_all_panda_serials = sorted(_all_pandas.keys()) - + with Panda(serial=None, claim=False) as p: + global _panda_type + global _panda_serial + _panda_serial = p.get_usb_serial() + _panda_type = bytes(p.get_type()) + assert _panda_serial is not None, "No panda found!" +init_devices() def init_jungle(): if _panda_jungle is None: @@ -90,24 +60,9 @@ def pytest_collection_modifyitems(items): if item.get_closest_marker('timeout') is None: item.add_marker(pytest.mark.timeout(60)) - # xdist grouping by panda - serial = item.name.split("serial=")[1].split(",")[0] - assert len(serial) == 24 - item.add_marker(pytest.mark.xdist_group(serial)) - needs_jungle = "panda_jungle" in item.fixturenames - if PARALLEL and needs_jungle: - item.add_marker(pytest.mark.skip(reason="no jungle tests in PARALLEL mode")) - elif NON_PARALLEL and not needs_jungle: - item.add_marker(pytest.mark.skip(reason="only running jungle tests")) - -def pytest_make_parametrize_id(config, val, argname): - if val in _all_pandas: - # TODO: get nice string instead of int - hw_type = _all_pandas[val][0] - return f"serial={val}, hw_type={hw_type}" - return None - + if needs_jungle and NO_JUNGLE: + item.add_marker(pytest.mark.skip(reason="skipping tests that requires a jungle")) @pytest.fixture(name='panda_jungle', scope='function') def fixture_panda_jungle(request): @@ -116,6 +71,8 @@ def fixture_panda_jungle(request): @pytest.fixture(name='p', scope='function') def func_fixture_panda(request, module_panda): + # *** Setup *** + p = module_panda # Check if test is applicable to this panda @@ -123,14 +80,14 @@ def func_fixture_panda(request, module_panda): if mark: assert len(mark.args) > 0, "Missing panda types argument in mark" test_types = mark.args[0] - if _all_pandas[p.get_usb_serial()] not in test_types: + if _panda_type not in test_types: pytest.skip(f"Not applicable, {test_types} pandas only") mark = request.node.get_closest_marker('skip_panda_types') if mark: assert len(mark.args) > 0, "Missing panda types argument in mark" skip_types = mark.args[0] - if _all_pandas[p.get_usb_serial()] in skip_types: + if _panda_type in skip_types: pytest.skip(f"Not applicable to {skip_types}") # this is 2+ seconds on USB pandas due to slow @@ -140,10 +97,10 @@ def func_fixture_panda(request, module_panda): # ensure FW hasn't changed assert p.up_to_date() - # Run test + # *** Run test *** yield p - # Teardown + # *** Teardown *** # reconnect if p.get_dfu_serial() in PandaDFU.list(): @@ -182,41 +139,27 @@ def func_fixture_panda(request, module_panda): assert can_health['total_error_cnt'] == 0 assert can_health['total_tx_checksum_error_cnt'] == 0 -@pytest.fixture(name='module_panda', params=_all_panda_serials, scope='module') +@pytest.fixture(name='module_panda', scope='module') def fixture_panda_setup(request): """ - Clean up all pandas + jungle and return the panda under test. + Clean up panda + jungle and return the panda under test. """ - panda_serial = request.param - - # Initialize jungle + # init jungle init_jungle() - # Connect to pandas - def cnnct(s): - if s == panda_serial: - p = Panda(serial=s) - p.reset(reconnect=True) + # init panda + p = Panda(serial=_panda_serial) + p.reset(reconnect=True) - p.set_can_loopback(False) - p.set_power_save(False) - for bus, speed in BUS_SPEEDS: - p.set_can_speed_kbps(bus, speed) - clear_can_buffers(p) - p.set_power_save(False) - return p - elif not PARALLEL: - with Panda(serial=s) as p: - p.reset(reconnect=False) - return None - - with concurrent.futures.ThreadPoolExecutor() as exc: - ps = list(exc.map(cnnct, _all_panda_serials, timeout=20)) - pandas = [p for p in ps if p is not None] + p.set_can_loopback(False) + p.set_power_save(False) + for bus, speed in BUS_SPEEDS: + p.set_can_speed_kbps(bus, speed) + clear_can_buffers(p) + p.set_power_save(False) # run test - yield pandas[0] + yield p - # Teardown - for p in pandas: - p.close() + # teardown + p.close() diff --git a/tests/hitl/run_parallel_tests.sh b/tests/hitl/run_parallel_tests.sh deleted file mode 100755 index f60d838c..00000000 --- a/tests/hitl/run_parallel_tests.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -e - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -cd $DIR - -# n = number of pandas tested -PARALLEL=1 pytest --durations=0 *.py -n 5 --dist loadgroup -x diff --git a/tests/hitl/run_serial_tests.sh b/tests/hitl/run_serial_tests.sh deleted file mode 100755 index 4feae5e3..00000000 --- a/tests/hitl/run_serial_tests.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -set -e - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -cd $DIR - -NON_PARALLEL=1 pytest --durations=0 *.py -x