CI: run HITL tests on dos (#1304)

* run on dos

* skip voltage on dos

* run these

* revert that

---------

Co-authored-by: Comma Device <device@comma.ai>

Jenkinsfile

@@ -83,6 +83,7 @@ pipeline {
             phone_steps("panda-dos", [
               ["build", "scons -j4"],
               ["flash", "cd tests/ && ./ci_reset_internal_hw.py"],
+              ["test", "cd tests/hitl && HW_TYPES=6 pytest --durations=0 [2-6]*.py -k 'not test_send_recv'"],
             ])
           }
         }

tests/hitl/2_health.py

@@ -30,6 +30,7 @@ def test_orientation_detection(p, panda_jungle):
       assert False
     seen_orientations.append(detected_harness_orientation)
 
+@pytest.mark.skip_panda_types((Panda.HW_TYPE_DOS, ))
 def test_voltage(p):
   for _ in range(10):
     voltage = p.health()['voltage']

tests/hitl/conftest.py

@@ -17,10 +17,11 @@ PEDAL_SERIAL = 'none'
 JUNGLE_SERIAL = os.getenv("PANDAS_JUNGLE")
 PANDAS_EXCLUDE = os.getenv("PANDAS_EXCLUDE", "").strip().split(" ")
 PARTIAL_TESTS = os.environ.get("PARTIAL_TESTS", "0") == "1"
+HW_TYPES = os.environ.get("HW_TYPES", None)
 
 class PandaGroup:
-  H7 = (Panda.HW_TYPE_RED_PANDA, Panda.HW_TYPE_RED_PANDA_V2)
-  GEN2 = (Panda.HW_TYPE_BLACK_PANDA, Panda.HW_TYPE_UNO) + H7
+  H7 = (Panda.HW_TYPE_RED_PANDA, Panda.HW_TYPE_RED_PANDA_V2, Panda.HW_TYPE_TRES)
+  GEN2 = (Panda.HW_TYPE_BLACK_PANDA, Panda.HW_TYPE_UNO, Panda.HW_TYPE_DOS) + H7
   GPS = (Panda.HW_TYPE_BLACK_PANDA, Panda.HW_TYPE_UNO)
   GMLAN = (Panda.HW_TYPE_WHITE_PANDA, Panda.HW_TYPE_GREY_PANDA)
 
@@ -31,6 +32,9 @@ if PARTIAL_TESTS:
   # * red panda covers GEN2, STM32H7
   # * black panda covers STM32F4, GEN2, and GPS
   PandaGroup.TESTED = (Panda.HW_TYPE_BLACK_PANDA, Panda.HW_TYPE_RED_PANDA)  # type: ignore
+elif HW_TYPES is not None:
+  PandaGroup.TESTED = [bytes([int(x), ]) for x in HW_TYPES.strip().split(",")] # type: ignore
+
 
 # Find all pandas connected
 _all_pandas = {}
@@ -68,7 +72,10 @@ def init_jungle():
 
 def pytest_configure(config):
   config.addinivalue_line(
-    "markers", "test_panda_types(name): mark test to run only on specified panda types"
+    "markers", "test_panda_types(name): whitelist a test for specific panda types"
+  )
+  config.addinivalue_line(
+    "markers", "skip_panda_types(name): blacklist panda types from a test"
   )
   config.addinivalue_line(
     "markers", "panda_expect_can_error: mark test to ignore CAN health errors"
@@ -84,7 +91,7 @@ def pytest_make_parametrize_id(config, val, argname):
 
 
 @pytest.fixture(name='panda_jungle')
-def fixture__panda_jungle(request):
+def fixture_panda_jungle(request):
   init_jungle()
   return _panda_jungle
 
@@ -95,11 +102,18 @@ def func_fixture_panda(request, module_panda):
   # Check if test is applicable to this panda
   mark = request.node.get_closest_marker('test_panda_types')
   if mark:
-    assert len(mark.args) > 0, "Missing allowed panda types in mark"
+    assert len(mark.args) > 0, "Missing panda types argument in mark"
     test_types = mark.args[0]
     if _all_pandas[p.get_usb_serial()] not in test_types:
       pytest.skip(f"Not applicable, {test_types} pandas only")
 
+  mark = request.node.get_closest_marker('skip_panda_types')
+  if mark:
+    assert len(mark.args) > 0, "Missing panda types argument in mark"
+    skip_types = mark.args[0]
+    if _all_pandas[p.get_usb_serial()] in skip_types:
+      pytest.skip(f"Not applicable to {skip_types}")
+
   # TODO: reset is slow (2+ seconds)
   p.reset()
 

tests/hitl/helpers.py

@@ -2,16 +2,17 @@ import time
 import random
 
 
-def time_many_sends(p, bus, p_recv=None, msg_count=100, msg_id=None, two_pandas=False):
+def time_many_sends(p, bus, p_recv=None, msg_count=100, two_pandas=False):
   if p_recv is None:
     p_recv = p
-  if msg_id is None:
-    msg_id = random.randint(0x100, 0x200)
   if p == p_recv and two_pandas:
     raise ValueError("Cannot have two pandas that are the same panda")
 
+  msg_id = random.randint(0x100, 0x200)
+  to_send = [(msg_id, 0, b"\xaa" * 8, bus)] * msg_count
+
   start_time = time.monotonic()
-  p.can_send_many([(msg_id, 0, b"\xaa" * 8, bus)] * msg_count)
+  p.can_send_many(to_send)
   r = []
   r_echo = []
   r_len_expected = msg_count if two_pandas else msg_count * 2