Pytest: consistent codecov (#30408)

* seed only

* random seed

* ignore version.py

* increase max examples

* increase default max examples

.github/workflows/selfdrive_tests.yaml

@@ -27,7 +27,7 @@ env:
   RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c
 
   UNIT_TEST: coverage run --append -m unittest discover
-  PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5
+  PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0
 
 jobs:
   build_release:

codecov.yml

@@ -8,4 +8,5 @@ coverage:
 
 ignore:
   - "**/test_*.py"
-  - "selfdrive/test/**"
+  - "selfdrive/test/**"
+  - "system/version.py" # codecov changes depending on if we are in a branch or not

conftest.py

@@ -1,5 +1,6 @@
 import os
 import pytest
+import random
 
 from openpilot.common.prefix import OpenpilotPrefix
 from openpilot.system.hardware import TICI
@@ -9,6 +10,8 @@ from openpilot.system.hardware import TICI
 def openpilot_function_fixture():
   starting_env = dict(os.environ)
 
+  random.seed(0)
+
   # setup a clean environment for each test
   with OpenpilotPrefix():
     prefix = os.environ["OPENPILOT_PREFIX"]

selfdrive/car/tests/test_car_interfaces.py

@@ -18,7 +18,7 @@ from openpilot.selfdrive.test.fuzzy_generation import DrawType, FuzzyGenerator
 
 ALL_ECUS = list({ecu for ecus in FW_VERSIONS.values() for ecu in ecus.keys()})
 
-MAX_EXAMPLES = int(os.environ.get('MAX_EXAMPLES', '5'))
+MAX_EXAMPLES = int(os.environ.get('MAX_EXAMPLES', '20'))
 
 
 def get_fuzzy_car_interface_args(draw: DrawType) -> dict: