CI: use pytest for unittesting (#29709)

* switch to pytest

* static analysis

* fix the helpers

* static analysis

* remove that

* more parallel test

* cleanup

* static analysis

* durations min and tolerance

* reduce those iterations

* sleep time could be decreased too

* in case it's negative

* try using caching

* remove parallelization for first PR

* PR Cleanup

* reduce diff

* fix that

* bump panda
old-commit-hash: ca435d344714f08276f9f9da5039f730987f31fa

.github/workflows/selfdrive_tests.yaml

@@ -26,6 +26,7 @@ env:
   RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c
 
   UNIT_TEST: coverage run --append -m unittest discover
+  PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5
 
 jobs:
   build_release:
@@ -248,22 +249,9 @@ jobs:
       timeout-minutes: 40
       run: |
         ${{ env.RUN }} "export SKIP_LONG_TESTS=1 && \
-                        $UNIT_TEST common && \
-                        $UNIT_TEST selfdrive/boardd && \
-                        $UNIT_TEST selfdrive/controls && \
-                        $UNIT_TEST selfdrive/monitoring && \
-                        $UNIT_TEST system/loggerd && \
-                        $UNIT_TEST selfdrive/car && \
-                        $UNIT_TEST selfdrive/locationd && \
-                        $UNIT_TEST selfdrive/test/longitudinal_maneuvers && \
-                        $UNIT_TEST system/tests && \
-                        $UNIT_TEST system/ubloxd && \
+                        $PYTEST --rootdir . -c selfdrive/test/pytest-ci.ini && \
                         selfdrive/locationd/test/_test_locationd_lib.py && \
                         ./system/ubloxd/tests/test_glonass_runner && \
-                        $UNIT_TEST selfdrive/athena && \
-                        $UNIT_TEST selfdrive/thermald && \
-                        $UNIT_TEST system/hardware/tici && \
-                        $UNIT_TEST tools/lib/tests && \
                         ./selfdrive/ui/tests/create_test_translations.sh && \
                         QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
                         ./selfdrive/ui/tests/test_translations.py && \
@@ -274,8 +262,7 @@ jobs:
                         ./tools/replay/tests/test_replay && \
                         ./tools/cabana/tests/test_cabana && \
                         ./system/camerad/test/ae_gray_test && \
-                        ./selfdrive/test/process_replay/test_fuzzy.py && \
-                        coverage xml"
+                        ./selfdrive/test/process_replay/test_fuzzy.py"
     - name: "Upload coverage to Codecov"
       uses: codecov/codecov-action@v3
 
@@ -374,7 +361,7 @@ jobs:
     - name: Test car models
       timeout-minutes: 25
       run: |
-        ${{ env.RUN }} "pytest --cov --cov-report=xml -n auto --dist=loadscope selfdrive/car/tests/test_models.py && \
+        ${{ env.RUN }} "$PYTEST -n auto --dist=loadscope selfdrive/car/tests/test_models.py && \
                         chmod -R 777 /tmp/comma_download_cache"
       env:
         NUM_JOBS: 5

selfdrive/athena/tests/test_athenad.py

@@ -48,6 +48,7 @@ class TestAthenadMethods(unittest.TestCase):
       else:
         os.unlink(p)
 
+    dispatcher["listUploadQueue"]() # ensure queue is empty at start
 
   # *** test helpers ***
 

selfdrive/car/tests/test_fw_fingerprint.py

@@ -175,7 +175,7 @@ class TestFwFingerprint(unittest.TestCase):
 
 class TestFwFingerprintTiming(unittest.TestCase):
   N: int = 5
-  TOL: float = 0.1
+  TOL: float = 0.12
 
   @staticmethod
   def _run_thread(thread: threading.Thread) -> float:

selfdrive/test/pytest-ci.ini

@@ -0,0 +1,18 @@
+[pytest]
+testpaths = 
+    common
+    selfdrive/athena
+    selfdrive/boardd
+    selfdrive/car
+    selfdrive/controls
+    selfdrive/locationd
+    selfdrive/monitoring
+    selfdrive/thermald
+    selfdrive/test/longitudinal_maneuvers
+    system/hardware/tici
+    system/loggerd
+    system/tests
+    system/ubloxd
+    tools/lib/tests
+markers =
+    parallel: mark tests as parallelizable (tests with no global state, so can be run in parallel)

system/logmessaged.py

@@ -5,6 +5,7 @@ from typing import NoReturn
 import cereal.messaging as messaging
 from openpilot.common.logging_extra import SwagLogFileFormatter
 from openpilot.system.swaglog import get_file_handler
+from system.swaglog import SWAGLOG_IPC
 
 
 def main() -> NoReturn:
@@ -14,7 +15,7 @@ def main() -> NoReturn:
 
   ctx = zmq.Context.instance()
   sock = ctx.socket(zmq.PULL)
-  sock.bind("ipc:///tmp/logmessage")
+  sock.bind(f"ipc://{SWAGLOG_IPC}")
 
   # and we publish them
   log_message_sock = messaging.pub_sock('logMessage')

system/swaglog.py

@@ -15,6 +15,8 @@ if PC:
 else:
   SWAGLOG_DIR = "/data/log/"
 
+SWAGLOG_IPC = "/tmp/logmessage"
+
 def get_file_handler():
   Path(SWAGLOG_DIR).mkdir(parents=True, exist_ok=True)
   base_filename = os.path.join(SWAGLOG_DIR, "swaglog")
@@ -89,7 +91,7 @@ class UnixDomainSocketHandler(logging.Handler):
     self.zctx = zmq.Context()
     self.sock = self.zctx.socket(zmq.PUSH)
     self.sock.setsockopt(zmq.LINGER, 10)
-    self.sock.connect("ipc:///tmp/logmessage")
+    self.sock.connect(f"ipc://{SWAGLOG_IPC}")
     self.pid = os.getpid()
 
   def emit(self, record):

system/tests/test_logmessaged.py

@@ -7,7 +7,7 @@ import unittest
 import cereal.messaging as messaging
 from openpilot.selfdrive.manager.process_config import managed_processes
 from openpilot.system.swaglog import cloudlog, ipchandler
-from selfdrive.test.helpers import temporary_swaglog_dir
+from openpilot.selfdrive.test.helpers import temporary_swaglog_dir, temporary_swaglog_ipc
 
 
 class TestLogmessaged(unittest.TestCase):
@@ -35,6 +35,7 @@ class TestLogmessaged(unittest.TestCase):
     return list(glob.glob(os.path.join(self.temp_dir, "swaglog.*")))
 
   @temporary_swaglog_dir
+  @temporary_swaglog_ipc
   def test_simple_log(self, temp_dir):
     self._setup(temp_dir)
     msgs = [f"abc {i}" for i in range(10)]
@@ -46,6 +47,7 @@ class TestLogmessaged(unittest.TestCase):
     assert len(self._get_log_files()) >= 1
 
   @temporary_swaglog_dir
+  @temporary_swaglog_ipc
   def test_big_log(self, temp_dir):
     self._setup(temp_dir)
     n = 10

system/ubloxd/tests/test_ublox_processing.py

@@ -9,6 +9,7 @@ from laika.opt import calc_pos_fix
 from openpilot.selfdrive.test.openpilotci import get_url
 from openpilot.tools.lib.logreader import LogReader
 from openpilot.selfdrive.test.helpers import with_processes
+from openpilot.selfdrive.test.helpers import temporary_dir
 import cereal.messaging as messaging
 
 def get_gnss_measurements(log_reader):
@@ -54,8 +55,9 @@ class TestUbloxProcessing(unittest.TestCase):
     self.assertEqual(count_gps, 5036)
     self.assertEqual(count_glonass, 3651)
 
-  def test_get_fix(self):
-    dog = AstroDog()
+  @temporary_dir
+  def test_get_fix(self, temp_dir):
+    dog = AstroDog(cache_dir=temp_dir)
     position_fix_found = 0
     count_processed_measurements = 0
     count_corrected_measurements = 0