CI: add regen job (#30157)

* Ability to whitelist/blacklist cars in regen_all

* Add CI job for regen, running on 2 segments

* Run regen_all, not regen

* Use coverage run

* Add test_regen

* Use test_regen in ci test

* Add test case names

* ONNXCPU = 1

* Add mazda segment

* Use RUN_CL

* build cl image before running

* unset PYTHONWARNINGS

* Create regen cache

* Replace daemons with processes

* Skip ford

* Skip mazda

* Add comment about commented segments

* Update selfdrive/test/process_replay/test_regen.py

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>

* Remove unset pythonwarnings

---------

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>

.github/workflows/selfdrive_tests.yaml

@@ -228,6 +228,33 @@ jobs:
     - name: "Upload coverage to Codecov"
       uses: codecov/codecov-action@v3
 
+  regen:
+    name: regen
+    runs-on: 'ubuntu-20.04'
+    steps: 
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Cache test routes
+      id: dependency-cache
+      uses: actions/cache@v3
+      with:
+        path: .ci_cache/comma_download_cache
+        key: regen-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/test_regen.py') }}
+    - name: Build base Docker image
+      run: eval "$BUILD"
+    - name: Build Docker image
+      run: eval "$BUILD_CL"
+    - name: Build openpilot
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
+    - name: Run regen
+      timeout-minutes: 30
+      run: |
+        ${{ env.RUN_CL }} "ONNXCPU=1 $PYTEST -n auto --dist=loadscope selfdrive/test/process_replay/test_regen.py && \
+                           chmod -R 777 /tmp/comma_download_cache"
+
   test_modeld:
     name: model tests
     runs-on: ubuntu-20.04

selfdrive/test/process_replay/regen_all.py

@@ -29,15 +29,25 @@ def regen_job(segment, upload, disable_tqdm):
 
 
 if __name__ == "__main__":
+  all_cars = {car for car, _ in segments}
+
   parser = argparse.ArgumentParser(description="Generate new segments from old ones")
   parser.add_argument("-j", "--jobs", type=int, default=1)
   parser.add_argument("--no-upload", action="store_true")
+  parser.add_argument("--whitelist-cars", type=str, nargs="*", default=all_cars,
+                      help="Whitelist given cars from the test (e.g. HONDA)")
+  parser.add_argument("--blacklist-cars", type=str, nargs="*", default=[],
+                      help="Blacklist given cars from the test (e.g. HONDA)")
   args = parser.parse_args()
 
+  tested_cars = set(args.whitelist_cars) - set(args.blacklist_cars)
+  tested_cars = {c.upper() for c in tested_cars}
+  tested_segments = [(car, segment) for car, segment in segments if car in tested_cars]
+
   with concurrent.futures.ProcessPoolExecutor(max_workers=args.jobs) as pool:
-    p = pool.map(regen_job, segments, [not args.no_upload] * len(segments), [args.jobs > 1] * len(segments))
+    p = pool.map(regen_job, tested_segments, [not args.no_upload] * len(tested_segments), [args.jobs > 1] * len(tested_segments))
     msg = "Copy these new segments into test_processes.py:"
-    for seg in tqdm(p, desc="Generating segments", total=len(segments)):
+    for seg in tqdm(p, desc="Generating segments", total=len(tested_segments)):
       msg += "\n" + str(seg)
     print()
     print()

selfdrive/test/process_replay/test_regen.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+
+import unittest
+
+from parameterized import parameterized
+
+from openpilot.selfdrive.test.process_replay.regen import regen_segment
+from openpilot.selfdrive.test.process_replay.process_replay import check_openpilot_enabled, CONFIGS
+from openpilot.selfdrive.test.openpilotci import get_url
+from openpilot.tools.lib.logreader import LogReader
+from openpilot.tools.lib.framereader import FrameReader
+
+EXCLUDED_PROCESSES = {"dmonitoringd", "dmonitoringmodeld"}
+TESTED_SEGMENTS = [
+  ("PRIUS_C2", "0982d79ebb0de295|2021-01-04--17-13-21--13"), # TOYOTA PRIUS 2017:     NEO, pandaStateDEPRECATED, no peripheralState, sensorEventsDEPRECATED
+  # Enable these once regen on CI becomes faster or use them for different tests running controlsd in isolation
+  # ("MAZDA_C3", "bd6a637565e91581|2021-10-30--15-14-53--4"),  # MAZDA.CX9_2021:        TICI, incomplete managerState
+  # ("FORD_C3", "54827bf84c38b14f|2023-01-26--21-59-07--4"),   # FORD.BRONCO_SPORT_MK1: TICI
+]
+
+
+def ci_setup_data_readers(route, sidx):
+  lr = LogReader(get_url(route, sidx, "rlog"))
+  # dm disabled
+  frs = {
+    'roadCameraState': FrameReader(get_url(route, sidx, "fcamera")),
+  }
+  if next((True for m in lr if m.which() == "wideRoadCameraState"), False):
+    frs["wideRoadCameraState"] = FrameReader(get_url(route, sidx, "ecamera"))
+
+  return lr, frs
+
+
+class TestRegen(unittest.TestCase):
+  @parameterized.expand(TESTED_SEGMENTS)
+  def test_engaged(self, case_name, segment):
+    tested_procs = [p for p in CONFIGS if p.proc_name not in EXCLUDED_PROCESSES]
+
+    route, sidx = segment.rsplit("--", 1)
+    lr, frs = ci_setup_data_readers(route, sidx)
+    output_logs = regen_segment(lr, frs, processes=tested_procs, disable_tqdm=True)
+
+    engaged = check_openpilot_enabled(output_logs)
+    self.assertTrue(engaged, f"openpilot not engaged in {case_name}")
+
+
+if __name__=='__main__':
+  unittest.main()