From dcfb206a38eb5719b872f1b7a60c25681f6d734f Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 11 May 2024 14:24:28 -0700 Subject: [PATCH] boardd: SPI corruption test (#32404) * simple test * little more --------- Co-authored-by: Comma Device --- Jenkinsfile | 1 + selfdrive/boardd/spi.cc | 12 ++- .../boardd/tests/test_boardd_loopback.py | 87 +++++++++++-------- selfdrive/boardd/tests/test_boardd_spi.py | 72 +++++++++++++++ 4 files changed, 132 insertions(+), 40 deletions(-) create mode 100755 selfdrive/boardd/tests/test_boardd_spi.py diff --git a/Jenkinsfile b/Jenkinsfile index 9d12b77746..2e672e1a23 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -237,6 +237,7 @@ node { deviceStage("tizi", "tizi", ["UNSAFE=1"], [ ["build openpilot", "cd selfdrive/manager && ./build.py"], ["test boardd loopback", "SINGLE_PANDA=1 pytest selfdrive/boardd/tests/test_boardd_loopback.py"], + ["test boardd spi", "pytest selfdrive/boardd/tests/test_boardd_spi.py"], ["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py"], ["test amp", "pytest system/hardware/tici/tests/test_amplifier.py"], ["test hw", "pytest system/hardware/tici/tests/test_hardware.py"], diff --git a/selfdrive/boardd/spi.cc b/selfdrive/boardd/spi.cc index ecf32bef5d..66e6a0f0ab 100644 --- a/selfdrive/boardd/spi.cc +++ b/selfdrive/boardd/spi.cc @@ -301,7 +301,11 @@ int PandaSpiHandle::lltransfer(spi_ioc_transfer &t) { } if ((static_cast(rand()) / RAND_MAX) < err_prob && t.tx_buf != (uint64_t)NULL) { printf("corrupting TX\n"); - memset((uint8_t*)t.tx_buf, (uint8_t)(rand() % 256), rand() % (t.len+1)); + for (int i = 0; i < t.len; i++) { + if ((static_cast(rand()) / RAND_MAX) > 0.9) { + ((uint8_t*)t.tx_buf)[i] = (uint8_t)(rand() % 256); + } + } } } @@ -310,7 +314,11 @@ int PandaSpiHandle::lltransfer(spi_ioc_transfer &t) { if (err_prob > 0) { if ((static_cast(rand()) / RAND_MAX) < err_prob && t.rx_buf != (uint64_t)NULL) { printf("corrupting RX\n"); - memset((uint8_t*)t.rx_buf, (uint8_t)(rand() % 256), rand() % (t.len+1)); + for (int i = 0; i < t.len; i++) { + if ((static_cast(rand()) / RAND_MAX) > 0.9) { + ((uint8_t*)t.rx_buf)[i] = (uint8_t)(rand() % 256); + } + } } } diff --git a/selfdrive/boardd/tests/test_boardd_loopback.py b/selfdrive/boardd/tests/test_boardd_loopback.py index 148ce9a25d..3ab3a9c5b1 100755 --- a/selfdrive/boardd/tests/test_boardd_loopback.py +++ b/selfdrive/boardd/tests/test_boardd_loopback.py @@ -3,6 +3,7 @@ import os import copy import random import time +import pytest import unittest from collections import defaultdict from pprint import pprint @@ -17,42 +18,61 @@ from openpilot.system.hardware import TICI from openpilot.selfdrive.test.helpers import phone_only, with_processes -class TestBoardd(unittest.TestCase): +def setup_boardd(num_pandas): + params = Params() + params.put_bool("IsOnroad", False) + with Timeout(90, "boardd didn't start"): + sm = messaging.SubMaster(['pandaStates']) + while sm.recv_frame['pandaStates'] < 1 or len(sm['pandaStates']) == 0 or \ + any(ps.pandaType == log.PandaState.PandaType.unknown for ps in sm['pandaStates']): + sm.update(1000) + + found_pandas = len(sm['pandaStates']) + assert num_pandas == found_pandas, "connected pandas ({found_pandas}) doesn't match expected panda count ({num_pandas}). \ + connect another panda for multipanda tests." + + # boardd safety setting relies on these params + cp = car.CarParams.new_message() + + safety_config = car.CarParams.SafetyConfig.new_message() + safety_config.safetyModel = car.CarParams.SafetyModel.allOutput + cp.safetyConfigs = [safety_config]*num_pandas + + params.put_bool("IsOnroad", True) + params.put_bool("FirmwareQueryDone", True) + params.put_bool("ControlsReady", True) + params.put("CarParams", cp.to_bytes()) + + +def send_random_can_messages(sendcan, count, num_pandas=1): + sent_msgs = defaultdict(set) + for _ in range(count): + to_send = [] + for __ in range(random.randrange(20)): + bus = random.choice([b for b in range(3*num_pandas) if b % 4 != 3]) + addr = random.randrange(1, 1<<29) + dat = bytes(random.getrandbits(8) for _ in range(random.randrange(1, 9))) + if (addr, dat) in sent_msgs[bus]: + continue + sent_msgs[bus].add((addr, dat)) + to_send.append(make_can_msg(addr, dat, bus)) + sendcan.send(can_list_to_can_capnp(to_send, msgtype='sendcan')) + return sent_msgs + + +@pytest.mark.tici +class TestBoarddLoopback: @classmethod - def setUpClass(cls): + def setup_class(cls): os.environ['STARTED'] = '1' os.environ['BOARDD_LOOPBACK'] = '1' @phone_only @with_processes(['pandad']) def test_loopback(self): - params = Params() - params.put_bool("IsOnroad", False) - - with Timeout(90, "boardd didn't start"): - sm = messaging.SubMaster(['pandaStates']) - while sm.recv_frame['pandaStates'] < 1 or len(sm['pandaStates']) == 0 or \ - any(ps.pandaType == log.PandaState.PandaType.unknown for ps in sm['pandaStates']): - sm.update(1000) - - num_pandas = len(sm['pandaStates']) - expected_pandas = 2 if TICI and "SINGLE_PANDA" not in os.environ else 1 - self.assertEqual(num_pandas, expected_pandas, "connected pandas ({num_pandas}) doesn't match expected panda count ({expected_pandas}). \ - connect another panda for multipanda tests.") - - # boardd safety setting relies on these params - cp = car.CarParams.new_message() - - safety_config = car.CarParams.SafetyConfig.new_message() - safety_config.safetyModel = car.CarParams.SafetyModel.allOutput - cp.safetyConfigs = [safety_config]*num_pandas - - params.put_bool("IsOnroad", True) - params.put_bool("FirmwareQueryDone", True) - params.put_bool("ControlsReady", True) - params.put("CarParams", cp.to_bytes()) - + num_pandas = 2 if TICI and "SINGLE_PANDA" not in os.environ else 1 + setup_boardd(num_pandas) sendcan = messaging.pub_sock('sendcan') can = messaging.sub_sock('can', conflate=False, timeout=100) sm = messaging.SubMaster(['pandaStates']) @@ -62,16 +82,7 @@ class TestBoardd(unittest.TestCase): for i in range(n): print(f"boardd loopback {i}/{n}") - sent_msgs = defaultdict(set) - for _ in range(random.randrange(20, 100)): - to_send = [] - for __ in range(random.randrange(20)): - bus = random.choice([b for b in range(3*num_pandas) if b % 4 != 3]) - addr = random.randrange(1, 1<<29) - dat = bytes(random.getrandbits(8) for _ in range(random.randrange(1, 9))) - sent_msgs[bus].add((addr, dat)) - to_send.append(make_can_msg(addr, dat, bus)) - sendcan.send(can_list_to_can_capnp(to_send, msgtype='sendcan')) + sent_msgs = send_random_can_messages(sendcan, random.randrange(20, 100), num_pandas) sent_loopback = copy.deepcopy(sent_msgs) sent_loopback.update({k+128: copy.deepcopy(v) for k, v in sent_msgs.items()}) diff --git a/selfdrive/boardd/tests/test_boardd_spi.py b/selfdrive/boardd/tests/test_boardd_spi.py new file mode 100755 index 0000000000..d384caaddd --- /dev/null +++ b/selfdrive/boardd/tests/test_boardd_spi.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +import os +import time +import numpy as np +import pytest + +import cereal.messaging as messaging +from cereal.services import SERVICE_LIST +from openpilot.system.hardware import HARDWARE +from openpilot.selfdrive.test.helpers import phone_only, with_processes +from openpilot.selfdrive.boardd.tests.test_boardd_loopback import setup_boardd + + +@pytest.mark.tici +class TestBoarddSpi: + @classmethod + def setup_class(cls): + if HARDWARE.get_device_type() == 'tici': + pytest.skip("only for spi pandas") + os.environ['STARTED'] = '1' + os.environ['BOARDD_LOOPBACK'] = '1' + os.environ['SPI_ERR_PROB'] = '0.001' + + @phone_only + @with_processes(['pandad']) + def test_spi_corruption(self, subtests): + setup_boardd(1) + + socks = {s: messaging.sub_sock(s, conflate=False, timeout=100) for s in ('can', 'pandaStates', 'peripheralState')} + time.sleep(2) + for s in socks.values(): + messaging.drain_sock_raw(s) + + st = time.monotonic() + ts = {s: list() for s in socks.keys()} + for _ in range(20): + for service, sock in socks.items(): + for m in messaging.drain_sock(sock): + ts[service].append(m.logMonoTime) + + # sanity check for corruption + assert m.valid + if service == "can": + assert len(m.can) == 0 + elif service == "pandaStates": + assert len(m.pandaStates) == 1 + ps = m.pandaStates[0] + assert ps.uptime < 100 + assert ps.pandaType == "tres" + assert ps.ignitionLine + assert not ps.ignitionCan + assert ps.voltage < 14000 + elif service == "peripheralState": + ps = m.peripheralState + assert ps.pandaType == "tres" + assert 4000 < ps.voltage < 14000 + assert 100 < ps.current < 1000 + assert ps.fanSpeedRpm < 8000 + + time.sleep(0.5) + et = time.monotonic() - st + + print("\n======== timing report ========") + for service, times in ts.items(): + dts = np.diff(times)/1e6 + print(service.ljust(17), f"{np.mean(dts):7.2f} {np.min(dts):7.2f} {np.max(dts):7.2f}") + with subtests.test(msg="timing check", service=service): + edt = 1e3 / SERVICE_LIST[service].frequency + assert edt*0.9 < np.mean(dts) < edt*1.1 + assert np.max(dts) < edt*3 + assert np.min(dts) < edt + assert len(dts) >= ((et-0.5)*SERVICE_LIST[service].frequency*0.8)