openpilot0/conftest.py

101 lines
3.1 KiB
Python
Raw Normal View History

import contextlib
import gc
import os
import pytest
import random
from openpilot.common.prefix import OpenpilotPrefix
from openpilot.selfdrive.manager import manager
from openpilot.system.hardware import TICI, HARDWARE
def pytest_sessionstart(session):
# TODO: fix tests and enable test order randomization
if session.config.pluginmanager.hasplugin('randomly'):
session.config.option.randomly_reorganize = False
test_models: fuzz test panda and CarState (#30443) * pre-hypothesis * some hypothesis junk * this kinda works but is really slow due to counter check * choose addrs from fingerprint * stash * honda nidec brake pressed mismatches fixed * bump panda * stash * tesla: use DI_torque2 (panda msg) * run * run * ah this honda mismatch too * no more multi can msgs * clean up, remove old file * add todo * prob can remove urandom * stash, huge examples * fix pq standstill mismatch * yuge * yup there's a leak somewhere * try to find leak * skip dashcam (pq and tesla) * PR comments * bump * draft stash * fix alt brake hondas * bump * bump * bump * some clean up * minor clean up * more clean up * stash * fix honda bug * more * 100 examples * revert tesla * no memory leak any more? * bring back tests with skips * parameterize max_examples * skip interceptor * is jenkins on my branch? * ooh that's fast * 50 is not bad for GH CI * 300 might be better with rest of test_models * no more detection * bump * need CS_prev to catch bugs where openpilot changes and panda doesn't (eg. not setting interceptor safety mode) * need to simplify all this * need a warm up first, since some signals are 1 by default (toyota's gas_released!=1) * changes * set honda safety param * set toyota safety param * bump panda * clean up honda * rm interceptor * thought interleaving addrs might help, but we can fine tune later * Revert "thought interleaving addrs might help, but we can fine tune later" This reverts commit 153301384b48c9f33f9e2af3c224241eaeec41c1. * get size from dict * what * add nocapture marker * clean up * try to raise logging level * need to run last as pytest_runtest_call, since it starts capturing * get capman conditionally * mark * type fingerprint * should use gen_empty_fingerprint * no longer needed * draft * no longer need gc * clean that up * test everything! * more clean up * more * no point * fix that * fix errors * bump * nice even 300 examples for 300 segs * final bump :fingers_crossed: * better import order * remove debugging prints * warm up kinda works * Revert "warm up kinda works" This reverts commit 7fc77b07d592edb13eadca77deb49540954a7d69. * random seed * revert * strat strat * add expl comment * cmt * check controls allowed * Revert "check controls allowed" This reverts commit e82a0e5396810dd4670e6847aa555194a709e10f. * not unittests * run tests! * run tests 2! * run tests 3! * seed unused * revert * add shrink phase, and remove health check suppression * hello * oncemore * Update selfdrive/car/tests/test_models.py
2023-12-19 17:18:54 +08:00
@pytest.hookimpl(hookwrapper=True, trylast=True)
def pytest_runtest_call(item):
# ensure we run as a hook after capturemanager's
if item.get_closest_marker("nocapture") is not None:
capmanager = item.config.pluginmanager.getplugin("capturemanager")
with capmanager.global_and_fixture_disabled():
yield
else:
yield
@contextlib.contextmanager
def clean_env():
starting_env = dict(os.environ)
yield
os.environ.clear()
os.environ.update(starting_env)
@pytest.fixture(scope="function", autouse=True)
def openpilot_function_fixture(request):
random.seed(0)
with clean_env():
# setup a clean environment for each test
with OpenpilotPrefix(shared_download_cache=request.node.get_closest_marker("shared_download_cache") is not None) as prefix:
prefix = os.environ["OPENPILOT_PREFIX"]
yield
# ensure the test doesn't change the prefix
assert "OPENPILOT_PREFIX" in os.environ and prefix == os.environ["OPENPILOT_PREFIX"]
# cleanup any started processes
manager.manager_cleanup()
# some processes disable gc for performance, re-enable here
if not gc.isenabled():
gc.enable()
gc.collect()
# If you use setUpClass, the environment variables won't be cleared properly,
# so we need to hook both the function and class pytest fixtures
@pytest.fixture(scope="class", autouse=True)
def openpilot_class_fixture():
with clean_env():
yield
@pytest.fixture(scope="function")
def tici_setup_fixture(openpilot_function_fixture):
"""Ensure a consistent state for tests on-device. Needs the openpilot function fixture to run first."""
HARDWARE.initialize_hardware()
HARDWARE.set_power_save(False)
os.system("pkill -9 -f athena")
@pytest.hookimpl(tryfirst=True)
def pytest_collection_modifyitems(config, items):
skipper = pytest.mark.skip(reason="Skipping tici test on PC")
for item in items:
if "tici" in item.keywords:
if not TICI:
item.add_marker(skipper)
else:
item.fixturenames.append('tici_setup_fixture')
if "xdist_group_class_property" in item.keywords:
class_property_name = item.get_closest_marker('xdist_group_class_property').args[0]
class_property_value = getattr(item.cls, class_property_name)
item.add_marker(pytest.mark.xdist_group(class_property_value))
@pytest.hookimpl(trylast=True)
def pytest_configure(config):
test_models: fuzz test panda and CarState (#30443) * pre-hypothesis * some hypothesis junk * this kinda works but is really slow due to counter check * choose addrs from fingerprint * stash * honda nidec brake pressed mismatches fixed * bump panda * stash * tesla: use DI_torque2 (panda msg) * run * run * ah this honda mismatch too * no more multi can msgs * clean up, remove old file * add todo * prob can remove urandom * stash, huge examples * fix pq standstill mismatch * yuge * yup there's a leak somewhere * try to find leak * skip dashcam (pq and tesla) * PR comments * bump * draft stash * fix alt brake hondas * bump * bump * bump * some clean up * minor clean up * more clean up * stash * fix honda bug * more * 100 examples * revert tesla * no memory leak any more? * bring back tests with skips * parameterize max_examples * skip interceptor * is jenkins on my branch? * ooh that's fast * 50 is not bad for GH CI * 300 might be better with rest of test_models * no more detection * bump * need CS_prev to catch bugs where openpilot changes and panda doesn't (eg. not setting interceptor safety mode) * need to simplify all this * need a warm up first, since some signals are 1 by default (toyota's gas_released!=1) * changes * set honda safety param * set toyota safety param * bump panda * clean up honda * rm interceptor * thought interleaving addrs might help, but we can fine tune later * Revert "thought interleaving addrs might help, but we can fine tune later" This reverts commit 153301384b48c9f33f9e2af3c224241eaeec41c1. * get size from dict * what * add nocapture marker * clean up * try to raise logging level * need to run last as pytest_runtest_call, since it starts capturing * get capman conditionally * mark * type fingerprint * should use gen_empty_fingerprint * no longer needed * draft * no longer need gc * clean that up * test everything! * more clean up * more * no point * fix that * fix errors * bump * nice even 300 examples for 300 segs * final bump :fingers_crossed: * better import order * remove debugging prints * warm up kinda works * Revert "warm up kinda works" This reverts commit 7fc77b07d592edb13eadca77deb49540954a7d69. * random seed * revert * strat strat * add expl comment * cmt * check controls allowed * Revert "check controls allowed" This reverts commit e82a0e5396810dd4670e6847aa555194a709e10f. * not unittests * run tests! * run tests 2! * run tests 3! * seed unused * revert * add shrink phase, and remove health check suppression * hello * oncemore * Update selfdrive/car/tests/test_models.py
2023-12-19 17:18:54 +08:00
config_line = "xdist_group_class_property: group tests by a property of the class that contains them"
config.addinivalue_line("markers", config_line)
config_line = "nocapture: don't capture test output"
config.addinivalue_line("markers", config_line)
config_line = "shared_download_cache: share download cache between tests"
config.addinivalue_line("markers", config_line)