safety: coverage report (#1696)

* coverage report

* test script

* not needed

* option

* options

* re-build if file missing

* rename

* rename flag

* just always build

* cmt

* fix rebuild, panda.c is what does gens gcno

* clean up

* check coverage

* rename

* add skip flag

* Revert "add skip flag"

This reverts commit 6991c174cebb8529b67199033e017155eff917b0.

* both do same thing

* clean up script

* don't test here

* Revert "don't test here"

This reverts commit 43fe4c961fcc1693d5efc33cabb8162be56a1b88.

* try running in CI

* nproc

* move

* Revert "try running in CI"

This reverts commit 912017c7fa93ba3faf31267b9b144cc9faee591b.

* don't test here

* fix path

* re-compiling does this

.gitignore

@@ -22,3 +22,9 @@ nosetests.xml
 
 # CTU info files generated by Cppcheck
 *.*.ctu-info
+
+# safety coverage-related files
+*.gcda
+*.gcno
+tests/safety/coverage-out
+tests/safety/coverage.info

SConstruct

@@ -8,5 +8,9 @@ AddOption('--ubsan',
           action='store_true',
           help='turn on UBSan')
 
+AddOption('--safety-coverage',
+          action='store_true',
+          help='build with safety test coverage options')
+
 # panda fw & test files
 SConscript('SConscript')

tests/libpanda/SConscript

@@ -29,4 +29,14 @@ if GetOption('ubsan'):
   env['CFLAGS'] += flags
   env['LINKFLAGS'] += flags
 
-env.SharedLibrary("libpanda.so", ["panda.c",])
+panda = env.SharedObject("panda.os", "panda.c")
+libpanda = env.SharedLibrary("libpanda.so", [panda])
+
+if GetOption('safety_coverage'):
+  env.Append(
+    CFLAGS=["-fprofile-arcs", "-ftest-coverage", "-fprofile-abs-path",],
+    LIBS=["gcov"],
+  )
+  # GCC note file is generated by compiler, ensure we build it, and allow scons to clean it up
+  AlwaysBuild(panda)
+  env.SideEffect("panda.gcno", panda)

tests/safety/test_coverage.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -e
+
+# reset coverage data and generate gcc note file
+scons -j$(nproc) -D --safety-coverage
+
+# run safety tests to generate coverage data
+./test.sh
+
+# generate and open report
+geninfo ../libpanda/ -o coverage.info
+genhtml coverage.info -o coverage-out
+browse coverage-out/index.html