small HITL test speedup (#1222)

* small speedup

* cleanup

* parallel connect

* little more

* less flaky

* update number of pandas

Co-authored-by: Bruce Wayne <batman@comma.ai>

tests/ci_reset_hw.py

@@ -16,7 +16,7 @@ if __name__ == "__main__":
 
   pandas = PandaDFU.list()
   print("DFU pandas:", pandas)
-  assert len(pandas) == 8
+  assert len(pandas) == 7
 
   with concurrent.futures.ProcessPoolExecutor(max_workers=len(pandas)) as exc:
     def recover(serial):
@@ -27,7 +27,7 @@ if __name__ == "__main__":
 
   pandas = Panda.list()
   print(pandas)
-  assert len(pandas) == 8
+  assert len(pandas) == 7
 
   with concurrent.futures.ProcessPoolExecutor(max_workers=len(pandas)) as exc:
     def flash(serial):

tests/hitl/1_program.py

@@ -6,7 +6,7 @@ from .helpers import test_all_pandas, panda_connect_and_init, check_signature
 
 
 @test_all_pandas
-@panda_connect_and_init(full_reset=False)
+@panda_connect_and_init
 def test_a_known_bootstub(p):
   # Test that compiled app can work with known production bootstub
   KNOWN_H7_BOOTSTUB_FN = os.path.join(BASEDIR, "tests", "hitl", "known_bootstub", "bootstub.panda_h7.bin")
@@ -29,13 +29,13 @@ def test_a_known_bootstub(p):
   check_signature(p)
 
 @test_all_pandas
-@panda_connect_and_init(full_reset=False)
+@panda_connect_and_init
 def test_b_recover(p):
   assert p.recover(timeout=30)
   check_signature(p)
 
 @test_all_pandas
-@panda_connect_and_init(full_reset=False)
+@panda_connect_and_init
 def test_c_flash(p):
   # test flash from bootstub
   serial = p._serial

tests/hitl/2_health.py

@@ -35,7 +35,7 @@ def test_orientation_detection(p):
     seen_orientations.append(detected_harness_orientation)
 
 @test_all_pandas
-@panda_connect_and_init(full_reset=False)
+@panda_connect_and_init
 def test_voltage(p):
   for _ in range(10):
     voltage = p.health()['voltage']

tests/hitl/3_usb_to_can.py

@@ -4,7 +4,7 @@ from flaky import flaky
 from nose.tools import assert_equal, assert_less, assert_greater
 
 from panda import Panda
-from .helpers import SPEED_NORMAL, SPEED_GMLAN, time_many_sends, test_white_and_grey, panda_type_to_serial, test_all_pandas, panda_connect_and_init
+from .helpers import SPEED_NORMAL, SPEED_GMLAN, time_many_sends, test_all_gmlan_pandas, test_all_pandas, panda_connect_and_init
 
 @test_all_pandas
 @panda_connect_and_init
@@ -67,7 +67,7 @@ def test_reliability(p):
     sys.stdout.flush()
 
 @test_all_pandas
-@flaky(max_runs=3, min_passes=1)
+@flaky(max_runs=6, min_passes=1)
 @panda_connect_and_init
 def test_throughput(p):
   # enable output mode
@@ -90,8 +90,7 @@ def test_throughput(p):
 
     print("loopback 100 messages at speed %d, comp speed is %.2f, percent %.2f" % (speed, comp_kbps, saturation_pct))
 
-@test_white_and_grey
-@panda_type_to_serial
+@test_all_gmlan_pandas
 @panda_connect_and_init
 def test_gmlan(p):
   p.set_safety_mode(Panda.SAFETY_ALLOUTPUT)
@@ -115,8 +114,7 @@ def test_gmlan(p):
 
     print("%d: %.2f kbps vs %.2f kbps" % (bus, comp_kbps_gmlan, comp_kbps_normal))
 
-@test_white_and_grey
-@panda_type_to_serial
+@test_all_gmlan_pandas
 @panda_connect_and_init
 def test_gmlan_bad_toggle(p):
   p.set_safety_mode(Panda.SAFETY_ALLOUTPUT)

tests/hitl/4_can_loopback.py

@@ -47,7 +47,7 @@ def test_send_recv(p):
 
 
 @test_all_pandas
-@flaky(max_runs=3, min_passes=1)
+@flaky(max_runs=6, min_passes=1)
 @panda_connect_and_init
 def test_latency(p):
   def test(p_send, p_recv):

tests/hitl/helpers.py

@@ -1,10 +1,11 @@
+import concurrent.futures
 import os
 import time
 import random
 import faulthandler
 from functools import wraps, partial
 from nose.tools import assert_equal
-from parameterized import parameterized, param
+from parameterized import parameterized
 
 from panda import Panda, DEFAULT_H7_FW_FN, DEFAULT_FW_FN, MCU_TYPE_H7
 from panda_jungle import PandaJungle  # pylint: disable=import-error
@@ -12,10 +13,9 @@ from panda_jungle import PandaJungle  # pylint: disable=import-error
 SPEED_NORMAL = 500
 SPEED_GMLAN = 33.3
 BUS_SPEEDS = [(0, SPEED_NORMAL), (1, SPEED_NORMAL), (2, SPEED_NORMAL), (3, SPEED_GMLAN)]
-TIMEOUT = 45
 H7_HW_TYPES = [Panda.HW_TYPE_RED_PANDA, Panda.HW_TYPE_RED_PANDA_V2]
 GEN2_HW_TYPES = [Panda.HW_TYPE_BLACK_PANDA, Panda.HW_TYPE_UNO] + H7_HW_TYPES
-GPS_HW_TYPES = [Panda.HW_TYPE_GREY_PANDA, Panda.HW_TYPE_BLACK_PANDA, Panda.HW_TYPE_UNO]
+GPS_HW_TYPES = [Panda.HW_TYPE_BLACK_PANDA, Panda.HW_TYPE_UNO]
 PEDAL_SERIAL = 'none'
 JUNGLE_SERIAL = os.getenv("PANDAS_JUNGLE")
 PANDAS_EXCLUDE = os.getenv("PANDAS_EXCLUDE", "").strip().split(" ")
@@ -41,33 +41,41 @@ def init_all_pandas():
   time.sleep(5)
 
   for serial in Panda.list():
-    if serial not in PANDAS_EXCLUDE:
+    if serial not in PANDAS_EXCLUDE and serial != PEDAL_SERIAL:
       with Panda(serial=serial) as p:
         _all_pandas.append((serial, p.get_type()))
-  print(f"Found {len(_all_pandas)} pandas")
+  print(f"{len(_all_pandas)} total pandas")
 init_all_pandas()
 _all_panda_serials = [x[0] for x in _all_pandas]
 
+def parameterized_panda_types(types):
+  serials = []
+  for typ in types:
+    for s, t in _all_pandas:
+      if t == typ and s not in serials:
+        serials.append(s)
+        break
+    else:
+      raise IOError("No unused panda found for type: {}".format(typ))
+  return parameterized(serials)
+
 # Panda providers
-test_pandas = _all_pandas[:]
+TESTED_HW_TYPES = (Panda.HW_TYPE_WHITE_PANDA, Panda.HW_TYPE_BLACK_PANDA, Panda.HW_TYPE_RED_PANDA, Panda.HW_TYPE_RED_PANDA_V2, Panda.HW_TYPE_UNO)
+test_all_pandas = parameterized_panda_types(TESTED_HW_TYPES)
+test_all_gen2_pandas = parameterized_panda_types(GEN2_HW_TYPES)
+test_all_gps_pandas = parameterized_panda_types(GPS_HW_TYPES)
+
+# no grey for speedup, should be sufficiently covered by white for these tests
+test_all_gmlan_pandas = parameterized_panda_types([Panda.HW_TYPE_WHITE_PANDA, ])
+
 if PARTIAL_TESTS:
   # minimal set of pandas to get most of our coverage
   # * red panda covers STM32H7
   # * black panda covers STM32F4, GEN2, and GPS
-  test_pandas = [p for p in _all_pandas if p[1] in (Panda.HW_TYPE_BLACK_PANDA, Panda.HW_TYPE_RED_PANDA)]
-test_all_pandas = parameterized(
-    list(map(lambda x: x[0], test_pandas))  # type: ignore
-  )
-test_all_gen2_pandas = parameterized(
-    list(map(lambda x: x[0], filter(lambda x: x[1] in GEN2_HW_TYPES, test_pandas)))  # type: ignore
-  )
-test_all_gps_pandas = parameterized(
-    list(map(lambda x: x[0], filter(lambda x: x[1] in GPS_HW_TYPES, test_pandas)))  # type: ignore
-  )
-test_white_and_grey = parameterized([
-    param(panda_type=Panda.HW_TYPE_WHITE_PANDA),
-    param(panda_type=Panda.HW_TYPE_GREY_PANDA)
-  ])
+  partial_pandas = (Panda.HW_TYPE_BLACK_PANDA, Panda.HW_TYPE_RED_PANDA)
+  test_all_pandas = parameterized_panda_types(partial_pandas)
+  test_all_gen2_pandas = parameterized_panda_types(partial_pandas)
+  test_all_gps_pandas = parameterized_panda_types([Panda.HW_TYPE_BLACK_PANDA, ])
 
 
 def time_many_sends(p, bus, p_recv=None, msg_count=100, msg_id=None, two_pandas=False):
@@ -107,35 +115,12 @@ def time_many_sends(p, bus, p_recv=None, msg_count=100, msg_id=None, two_pandas=
 
   return comp_kbps
 
-def panda_type_to_serial(fn):
-  @wraps(fn)
-  def wrapper(panda_type=None, **kwargs):
-    # Change panda_types to a list
-    if panda_type is not None:
-      if not isinstance(panda_type, list):
-        panda_type = [panda_type]
-
-    # Find a panda with the correct types and add the corresponding serial
-    serials = []
-    for p_type in panda_type:
-      found = False
-      for serial, pt in _all_pandas:
-        # Never take the same panda twice
-        if (pt == p_type) and (serial not in serials):
-          serials.append(serial)
-          found = True
-          break
-      if not found:
-        raise IOError("No unused panda found for type: {}".format(p_type))
-    return fn(serials, **kwargs)
-  return wrapper
-
-def panda_connect_and_init(fn=None, full_reset=True):
+def panda_connect_and_init(fn=None):
   if not fn:
-    return partial(panda_connect_and_init, full_reset=full_reset)
+    return partial(panda_connect_and_init)
 
   @wraps(fn)
-  def wrapper(panda_serials=None, **kwargs):
+  def wrapper(panda_serials, **kwargs):
     # Change panda_serials to a list
     if panda_serials is not None:
       if not isinstance(panda_serials, list):
@@ -148,7 +133,7 @@ def panda_connect_and_init(fn=None, full_reset=True):
     panda_jungle.set_obd(False)
     panda_jungle.set_harness_orientation(PandaJungle.HARNESS_ORIENTATION_1)
     for bus, speed in BUS_SPEEDS:
-        panda_jungle.set_can_speed_kbps(bus, speed)
+      panda_jungle.set_can_speed_kbps(bus, speed)
 
     # wait for all pandas to come up
     for _ in range(50):
@@ -157,27 +142,28 @@ def panda_connect_and_init(fn=None, full_reset=True):
       time.sleep(0.1)
 
     # Connect to pandas
-    pandas = []
-    for s in _all_panda_serials:
+    def cnnct(s):
       if s in panda_serials:
         p = Panda(serial=s)
         p.reset(reconnect=True)
-        pandas.append(p)
-      elif full_reset and s != PEDAL_SERIAL:
+
+        p.set_can_loopback(False)
+        p.set_gmlan(None)
+        p.set_esp_power(False)
+        p.set_power_save(False)
+        for bus, speed in BUS_SPEEDS:
+          p.set_can_speed_kbps(bus, speed)
+        clear_can_buffers(p)
+        p.set_power_save(False)
+        return p
+      else:
         with Panda(serial=s) as p:
           p.reset(reconnect=False)
+      return None
 
-    # Initialize pandas
-    if full_reset:
-      for panda in pandas:
-        panda.set_can_loopback(False)
-        panda.set_gmlan(None)
-        panda.set_esp_power(False)
-        panda.set_power_save(False)
-        for bus, speed in BUS_SPEEDS:
-          panda.set_can_speed_kbps(bus, speed)
-        clear_can_buffers(panda)
-        panda.set_power_save(False)
+    with concurrent.futures.ThreadPoolExecutor() as exc:
+      ps = list(exc.map(cnnct, _all_panda_serials, timeout=20))
+      pandas = [p for p in ps if p is not None]
 
     try:
       fn(*pandas, *kwargs)
@@ -185,12 +171,11 @@ def panda_connect_and_init(fn=None, full_reset=True):
       # Check if the pandas did not throw any faults while running test
       for panda in pandas:
         if not panda.bootstub:
-          panda.reconnect()
+          #panda.reconnect()
           assert panda.health()['fault_status'] == 0
           # Check health of each CAN core after test, normal to fail for test_gen2_loopback on OBD bus, so skipping
           if fn.__name__ != "test_gen2_loopback":
             for i in range(3):
-              print(fn.__name__)
               can_health = panda.can_health(i)
               assert can_health['bus_off_cnt'] == 0
               assert can_health['receive_error_cnt'] == 0