run all tests in test.sh (#2364)

.github/workflows/test.yaml

@@ -54,24 +54,3 @@ jobs:
         run: uv pip install --system .
       - name: Verify importing panda
         run: python -c "from panda import Panda"
-
-
-  misra_linter:
-    name: MISRA C:2012 Linter
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    steps:
-      - uses: actions/checkout@v4
-      - run: ./setup.sh
-      - name: Run MISRA C:2012 analysis
-        run: ${{ env.RUN }} "cd tests/misra && ./test_misra.sh"
-
-  misra_mutation:
-    name: MISRA C:2012 Mutation
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    steps:
-      - uses: actions/checkout@v4
-      - run: ./setup.sh
-      - name: MISRA mutation tests
-        run: ${{ env.RUN }} "cd tests/misra && pytest test_mutation.py"

pyproject.toml

@@ -27,11 +27,10 @@ dev = [
   "cffi",
   "flaky",
   "pytest",
+  "pytest-xdist",
   "pytest-mock",
   "pytest-timeout",
-  "pytest-randomly",
   "ruff",
-  "mypy",
   "setuptools",
   "gcc-arm-none-eabi @ git+https://github.com/commaai/dependencies.git@releases#subdirectory=gcc-arm-none-eabi",
   "cppcheck @ git+https://github.com/commaai/dependencies.git@releases#subdirectory=cppcheck",
@@ -53,18 +52,6 @@ packages = [
 [tool.setuptools.package-dir]
 panda = "."
 
-[tool.mypy]
-# third-party packages
-ignore_missing_imports = true
-
-# helpful warnings
-warn_redundant_casts = true
-warn_unreachable = true
-warn_unused_ignores = true
-
-# restrict dynamic typing
-warn_return_any = true
-
 # https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml
 [tool.ruff]
 line-length = 160
@@ -79,7 +66,7 @@ flake8-implicit-str-concat.allow-multiline=false
 "pytest.main".msg = "pytest.main requires special handling that is easy to mess up!"
 
 [tool.pytest.ini_options]
-addopts = "-Werror --strict-config --strict-markers --durations=10 --ignore-glob='*.sh' --ignore=tests/misra --ignore=tests/som --ignore=tests/hitl"
+addopts = "-Werror --strict-config --strict-markers --maxprocesses=8 -nauto --durations=10 --ignore-glob='*.sh' --ignore=tests/som --ignore=tests/hitl"
 python_files = "test_*.py"
 testpaths = [
   "tests/"

test.sh

@@ -10,12 +10,6 @@ source ./setup.sh
 # *** build ***
 scons
 
-# *** lint ***
+# *** lint + test ***
 ruff check .
-mypy python/
-
-
-# *** test ***
-
-# TODO: make randomly work
-pytest --randomly-dont-reorganize tests/
+pytest

tests/misra/test_mutation.py

@@ -10,6 +10,7 @@ import random
 HERE = os.path.abspath(os.path.dirname(__file__))
 ROOT = os.path.join(HERE, "../../")
 
+# skip mutating these paths
 IGNORED_PATHS = (
   'board/obj',
   'board/jungle',
@@ -53,14 +54,16 @@ patterns = [
 ]
 
 all_files = glob.glob('board/**', root_dir=ROOT, recursive=True)
-files = [f for f in all_files if f.endswith(('.c', '.h')) and not f.startswith(IGNORED_PATHS)]
+files = sorted(f for f in all_files if f.endswith(('.c', '.h')) and not f.startswith(IGNORED_PATHS))
 assert len(files) > 50, all(d in files for d in ('board/main.c', 'board/stm32h7/llfdcan.h'))
 
+# fixed seed so every xdist worker collects the same test params
+rng = random.Random(len(files))
 for p in patterns:
-  mutations.append((random.choice(files), p, True))
+  mutations.append((rng.choice(files), p, True))
 
-# TODO: remove sampling once test_misra.sh is faster
-mutations = random.sample(mutations, 2)
+# sample to keep CI fast, but always include the no-mutation case
+mutations = [mutations[0]] + rng.sample(mutations[1:], min(2, len(mutations) - 1))
 
 @pytest.mark.parametrize("fn, patch, should_fail", mutations)
 def test_misra_mutation(fn, patch, should_fail):
@@ -69,8 +72,16 @@ def test_misra_mutation(fn, patch, should_fail):
 
     # apply patch
     if fn is not None:
-      r = os.system(f"cd {tmp}/panda && sed -i '{patch}' {fn}")
-      assert r == 0
+      fpath = os.path.join(tmp, "panda", fn)
+      with open(fpath) as f:
+        content = f.read()
+      if patch.startswith("s/"):
+        old, new = patch[2:].rsplit("/g", 1)[0].split("/", 1)
+        content = content.replace(old, new)
+      elif patch.startswith("$a "):
+        content += patch[3:].replace(r"\n", "\n")
+      with open(fpath, "w") as f:
+        f.write(content)
 
     # run test
     r = subprocess.run("SKIP_TABLES_DIFF=1 panda/tests/misra/test_misra.sh", cwd=tmp, shell=True)

tests/usbprotocol/test_comms.py

@@ -31,6 +31,7 @@ def random_can_messages(n, bus=None):
 
 class TestPandaComms(unittest.TestCase):
   def setUp(self):
+    lpp.set_safety_hooks(CarParams.SafetyModel.allOutput, 0)
     lpp.comms_can_reset()
 
   def test_tx_queues(self):
@@ -102,8 +103,6 @@ class TestPandaComms(unittest.TestCase):
 
 
   def test_can_send_usb(self):
-    lpp.set_safety_hooks(CarParams.SafetyModel.allOutput, 0)
-
     for bus in range(3):
       with self.subTest(bus=bus):
         for _ in range(100):