From 61e987f6e280f2b84b6f5ccffaca91ae9635f411 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Mon, 7 Aug 2023 18:38:01 -0700 Subject: [PATCH] remove safety replay test (#1569) --- .github/workflows/test.yaml | 17 ------- tests/safety_replay/test_safety_replay.py | 59 ----------------------- 2 files changed, 76 deletions(-) delete mode 100755 tests/safety_replay/test_safety_replay.py diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cd18ec35..89da1bb4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -87,23 +87,6 @@ jobs: scons -j$(nproc) ${{ matrix.flags }} && \ tests/safety/test.sh" - safety_replay: - name: safety replay - runs-on: ubuntu-20.04 - strategy: - matrix: - flags: ['', '--ubsan'] - timeout-minutes: 20 - steps: - - uses: actions/checkout@v2 - - name: Build Docker image - run: eval "$BUILD" - - name: Run safety replay - run: | - ${{ env.RUN }} "scons -j$(nproc) ${{ matrix.flags }} && \ - cd tests/safety_replay && \ - ./test_safety_replay.py" - misra: name: misra c2012 runs-on: ubuntu-20.04 diff --git a/tests/safety_replay/test_safety_replay.py b/tests/safety_replay/test_safety_replay.py deleted file mode 100755 index 2dcc07cd..00000000 --- a/tests/safety_replay/test_safety_replay.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 -from collections import namedtuple -import os -import requests - -from panda import Panda -from panda.python import ALTERNATIVE_EXPERIENCE as ALT_EXP -from panda.tests.safety_replay.replay_drive import replay_drive -from tools.lib.logreader import LogReader # pylint: disable=import-error - -BASE_URL = "https://commadataci.blob.core.windows.net/openpilotci/" - -ReplayRoute = namedtuple("ReplayRoute", ("route", "safety_mode", "param", "alternative_experience"), defaults=(0, 0)) - -logs = [ - ReplayRoute("2425568437959f9d|2019-12-22--16-24-37.bz2", Panda.SAFETY_HONDA_NIDEC), # HONDA.CIVIC (fcw presents: 0x1FA blocked as expected) - ReplayRoute("38bfd238edecbcd7|2019-06-07--10-15-25.bz2", Panda.SAFETY_TOYOTA, 66), # TOYOTA.PRIUS - ReplayRoute("f89c604cf653e2bf|2018-09-29--13-46-50.bz2", Panda.SAFETY_GM), # GM.VOLT - ReplayRoute("6fb4948a7ebe670e|2019-11-12--00-35-53.bz2", Panda.SAFETY_CHRYSLER), # CHRYSLER.PACIFICA_2018_HYBRID - ReplayRoute("0744286ead2fbb96|2023-05-01--16-27-01--35--rlog.bz2", Panda.SAFETY_SUBARU), # SUBARU.IMPREZA - ReplayRoute("bad6ae3584ece5b5|2023-04-29--11-23-48--7--rlog.bz2", Panda.SAFETY_SUBARU, # SUBARU.OUTBACK - Panda.FLAG_SUBARU_GEN2, ALT_EXP.DISABLE_DISENGAGE_ON_GAS), - ReplayRoute("76b83eb0245de90e|2020-03-05--19-16-05.bz2", Panda.SAFETY_VOLKSWAGEN_MQB), # VOLKSWAGEN.GOLF (stock ACC) - ReplayRoute("3cfdec54aa035f3f|2022-10-13--14-58-58.bz2", Panda.SAFETY_VOLKSWAGEN_MQB, # VOLKSWAGEN.GOLF (openpilot long) - Panda.FLAG_VOLKSWAGEN_LONG_CONTROL), - ReplayRoute("3cfdec54aa035f3f|2022-07-19--23-45-10.bz2", Panda.SAFETY_VOLKSWAGEN_PQ, # VOLKSWAGEN.PASSAT_NMS (openpilot longitudinal) - Panda.FLAG_VOLKSWAGEN_LONG_CONTROL), - ReplayRoute("fbbfa6af821552b9|2020-03-03--08-09-43.bz2", Panda.SAFETY_NISSAN), # NISSAN.XTRAIL - ReplayRoute("5b7c365c50084530_2020-04-15--16-13-24--3--rlog.bz2", Panda.SAFETY_HYUNDAI), # HYUNDAI.SONATA - ReplayRoute("610ebb9faaad6b43|2020-06-13--15-28-36.bz2", Panda.SAFETY_HYUNDAI_LEGACY), # HYUNDAI.IONIQ_EV_LTD - ReplayRoute("5ab784f361e19b78_2020-06-08--16-30-41.bz2", Panda.SAFETY_SUBARU_PREGLOBAL), # SUBARU.OUTBACK_PREGLOBAL - ReplayRoute("bb50caf5f0945ab1|2021-06-19--17-20-18.bz2", Panda.SAFETY_TESLA), # TESLA.AP2_MODELS - ReplayRoute("bd6a637565e91581_2021-10-29--22-18-31--1--rlog.bz2", Panda.SAFETY_MAZDA), # MAZDA.CX9_2021 - ReplayRoute("1a5d045d2c531a6d_2022-06-07--22-03-00--1--rlog.bz2", Panda.SAFETY_HONDA_BOSCH, # HONDA.CIVIC_2022 - Panda.FLAG_HONDA_RADARLESS, ALT_EXP.DISABLE_DISENGAGE_ON_GAS), -] - - -if __name__ == "__main__": - - # get all the routes - for route, _, _, _ in logs: - if not os.path.isfile(route): - with open(route, "wb") as f: - r = requests.get(BASE_URL + route, timeout=10) - r.raise_for_status() - f.write(r.content) - - failed = [] - for route, mode, param, alt_exp in logs: - lr = LogReader(route, sort_by_time=True) - - print("\nreplaying %s with safety mode %d, param %s, alternative experience %s" % (route, mode, param, alt_exp)) - if not replay_drive(lr, mode, param, alt_exp): - failed.append(route) - - for f in failed: - print(f"\n**** failed on {f} ****") - assert len(failed) == 0, "\nfailed on %d logs" % len(failed)