From 5fe1d67b7729c843df7dd15df5d375d08ed410c6 Mon Sep 17 00:00:00 2001
From: Shane Smiskol <shane@smiskol.com>
Date: Fri, 10 Nov 2023 19:37:41 -0800
Subject: [PATCH] safety: test coverage (#1699)

* run in CI

* test

* make report optional

* 100% GM coverage

comments

* more coverage: these are unhittable lines

* vw mqb: convert switch to if, consistent with all other safety modes

* we don't check any of these yet

* rm

* ensure honda_fwd_brake is reset

* can rm this

* test honda brake latching

* honda: add rx brake function

* use loops for more coverage and more compact code

* other honda stuff

* remove car-specific stuff from init_tests

* don't need to have car safety modes reset interceptor detected (global init does)

* use get_bit

* ahh ford counter is unhittable: counter check disabled due to skipping

* misra

* test nooutput and alloutput

* required changes for that

* fix test

* more all/nooutput coverage

* start on lin

* rx coverage

* lin tx coverage

* some barebones body test

* double negative

* draft elm327 safety (git stash)

* fixes from merge

* clean up test cov

* add rm

* no body

* failed grep returns exit code 1

* more clear msgs
---
 .github/workflows/test.yaml   | 17 +++++++++++++++++
 tests/safety/test_coverage.sh | 20 +++++++++++++++++---
 2 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index fad852a0..ccd2877c 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -90,6 +90,23 @@ jobs:
                           scons -j$(nproc) ${{ matrix.flags }} && \
                           tests/safety/test.sh"
 
+  safety_coverage:
+    name: safety coverage
+    runs-on: ubuntu-20.04
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build Docker image
+        run: eval "$BUILD"
+      - name: Run safety coverage test
+        timeout-minutes: 5
+        run: |
+          ${{ env.RUN }} "cd .. && \
+                          scons -c && \
+                          scons -j$(nproc) opendbc/ cereal/ && \
+                          cd panda/tests/safety && \
+                          ./test_coverage.sh"
+
   misra:
     name: misra c2012
     runs-on: ubuntu-20.04
diff --git a/tests/safety/test_coverage.sh b/tests/safety/test_coverage.sh
index 469b9e16..c5168004 100755
--- a/tests/safety/test_coverage.sh
+++ b/tests/safety/test_coverage.sh
@@ -2,12 +2,26 @@
 set -e
 
 # reset coverage data and generate gcc note file
+rm -f ../libpanda/*.gcda
 scons -j$(nproc) -D --coverage
 
 # run safety tests to generate coverage data
 ./test.sh
 
 # generate and open report
-geninfo ../libpanda/ -o coverage.info
-genhtml coverage.info -o coverage-out
-browse coverage-out/index.html
+if [ "$1" == "--report" ]; then
+  geninfo ../libpanda/ -o coverage.info
+  genhtml coverage.info -o coverage-out
+  browse coverage-out/index.html
+fi
+
+# test coverage
+GCOV_OUTPUT=$(gcov -n ../libpanda/panda.c)
+INCOMPLETE_COVERAGE=$(echo "$GCOV_OUTPUT" | paste -s -d' \n' | grep "File.*safety/safety_.*.h" | grep -v "100.00%" || true)
+if [ -n "$INCOMPLETE_COVERAGE" ]; then
+  echo "FAILED: Some files have less than 100% coverage:"
+  echo "$INCOMPLETE_COVERAGE"
+  exit 1
+else
+  echo "SUCCESS: All checked files have 100% coverage!"
+fi