HITL tests: add timeouts (#1352)

* set timeout

* closest

* more timeout

* fan test is slow

* retry

* break

* bump

requirements.txt

@@ -4,6 +4,7 @@ hexdump>=3.3
 pycryptodome==3.9.8
 tqdm>=4.14.0
 pytest
+pytest-timeouts
 parameterized
 requests
 flake8==3.7.9

tests/hitl/1_program.py

@@ -1,5 +1,6 @@
 import os
 import time
+import pytest
 
 from panda import Panda, PandaDFU, McuType, BASEDIR
 
@@ -10,6 +11,7 @@ def check_signature(p):
 
 # TODO: make more comprehensive bootstub tests and run on a few production ones + current
 # TODO: also test release-signed app
+@pytest.mark.execution_timeout(30)
 def test_a_known_bootstub(p):
   """
   Test that compiled app can work with known production bootstub
@@ -55,10 +57,12 @@ def test_a_known_bootstub(p):
     check_signature(p)
     assert not p.bootstub
 
+@pytest.mark.execution_timeout(15)
 def test_b_recover(p):
   assert p.recover(timeout=30)
   check_signature(p)
 
+@pytest.mark.execution_timeout(25)
 def test_c_flash(p):
   # test flash from bootstub
   serial = p._serial

tests/hitl/4_can_loopback.py

@@ -11,6 +11,7 @@ from panda.tests.hitl.conftest import PandaGroup, PARTIAL_TESTS
 from panda.tests.hitl.helpers import time_many_sends, clear_can_buffers
 
 @flaky(max_runs=3, min_passes=1)
+@pytest.mark.execution_timeout(35)
 def test_send_recv(p, panda_jungle):
   def test(p_send, p_recv):
     p_send.set_can_loopback(False)
@@ -45,6 +46,7 @@ def test_send_recv(p, panda_jungle):
 
 
 @flaky(max_runs=6, min_passes=1)
+@pytest.mark.execution_timeout(30)
 def test_latency(p, panda_jungle):
   def test(p_send, p_recv):
     p_send.set_can_loopback(False)

tests/hitl/7_internal.py

@@ -8,6 +8,7 @@ pytestmark = [
   pytest.mark.test_panda_types(Panda.INTERNAL_DEVICES)
 ]
 
+@pytest.mark.execution_timeout(50)
 def test_fan_controller(p):
   start_health = p.health()
 

tests/hitl/conftest.py

@@ -87,6 +87,14 @@ def pytest_configure(config):
     "markers", "panda_expect_can_error: mark test to ignore CAN health errors"
   )
 
+def pytest_collection_modifyitems(items):
+  for item in items:
+    if item.get_closest_marker('execution_timeout') is None:
+      item.add_marker(pytest.mark.execution_timeout(10))
+
+    item.add_marker(pytest.mark.setup_timeout(18))
+    item.add_marker(pytest.mark.teardown_timeout(12))
+
 
 def pytest_make_parametrize_id(config, val, argname):
   if val in _all_pandas: