diff --git a/opendbc/safety/tests/libsafety/SConscript b/opendbc/safety/tests/libsafety/SConscript index a8cb50cc..3abd0375 100644 --- a/opendbc/safety/tests/libsafety/SConscript +++ b/opendbc/safety/tests/libsafety/SConscript @@ -1,16 +1,19 @@ import platform +CC = 'gcc' system = platform.system() +mac_ver = platform.mac_ver() -# Use clang-17 for both normal tests and mutation testing -# The Mull plugin injects mutations that are dormant unless run with mull-runner -if system == "Darwin": - mull_plugin = Dir('#').abspath + '/.mull/lib/mull-ir-frontend-17' -else: - mull_plugin = '/usr/lib/mull-ir-frontend-17' +# gcc installed by homebrew has version suffix (e.g. gcc-12) in order to be +# distinguishable from system one - which acts as a symlink to clang +# clang works on macOS 15 and greater but has issues on earlier macOS versions. +# see: https://github.com/commaai/openpilot/issues/35093 +if system == 'Darwin' and mac_ver[0] and mac_ver[0] < '15': + CC += '-13' +# standard env env = Environment( - CC='clang-17', + CC=CC, CFLAGS=[ '-Wall', "-Wextra", @@ -25,8 +28,6 @@ env = Environment( '-fno-omit-frame-pointer', '-fprofile-arcs', '-ftest-coverage', - f'-fpass-plugin={mull_plugin}', - '-grecord-command-line', '-DALLOW_DEBUG', ], LINKFLAGS=[ @@ -39,10 +40,31 @@ env = Environment( tools=["default", "compilation_db"], ) if system == "Darwin": - env.PrependENVPath('PATH', '/opt/homebrew/opt/llvm@17/bin') + env.PrependENVPath('PATH', '/opt/homebrew/bin') -safety = env.SharedObject("safety.os", "safety.c") -libsafety = env.SharedLibrary("libsafety.so", [safety]) +# mutation env +menv = env.Clone() +if system == "Darwin": + menv.PrependENVPath('PATH', '/opt/homebrew/opt/llvm@17/bin') + mull_plugin = Dir('#').abspath + '/.mull/lib/mull-ir-frontend-17' +else: + mull_plugin = '/usr/lib/mull-ir-frontend-17' +menv['CC'] = 'clang-17' +flags = [ + '-fprofile-instr-generate', + '-fcoverage-mapping', + f'-fpass-plugin={mull_plugin}', + '-g', + '-grecord-command-line', +] +menv['CFLAGS'] += flags +menv['LINKFLAGS'] = flags +menv['CFLAGS'].remove('-fprofile-arcs') +menv['CFLAGS'].remove('-ftest-coverage') -# GCC-style note file is generated by compiler, allow scons to clean it up -env.SideEffect("safety.gcno", safety) +for build_env, suffix in ((env, ""), (menv, "_mutation")): + safety = build_env.SharedObject(f"safety{suffix}.os", "safety.c") + libsafety = build_env.SharedLibrary(f"libsafety{suffix}.so", [safety]) + + # GCC note file is generated by compiler, allow scons to clean it up + build_env.SideEffect("safety.gcno", safety) diff --git a/opendbc/safety/tests/libsafety/libsafety_py.py b/opendbc/safety/tests/libsafety/libsafety_py.py index e93b782e..ee314b6e 100644 --- a/opendbc/safety/tests/libsafety/libsafety_py.py +++ b/opendbc/safety/tests/libsafety/libsafety_py.py @@ -4,7 +4,7 @@ from cffi import FFI from opendbc.safety import LEN_TO_DLC libsafety_dir = os.path.dirname(os.path.abspath(__file__)) -libsafety_fn = os.path.join(libsafety_dir, "libsafety.so") +libsafety_fn = os.path.join(libsafety_dir, "libsafety_mutation.so" if "MUTATION" in os.environ else "libsafety.so") ffi = FFI() diff --git a/opendbc/safety/tests/mutation.sh b/opendbc/safety/tests/mutation.sh index 411bab35..c0513357 100755 --- a/opendbc/safety/tests/mutation.sh +++ b/opendbc/safety/tests/mutation.sh @@ -17,4 +17,5 @@ EOF scons -j4 -D -mull-runner-17 --debug --ld-search-path /lib/x86_64-linux-gnu/ ./libsafety/libsafety.so -test-program=pytest -- -n8 --ignore-glob=misra/* \ No newline at end of file +export MUTATION=1 +mull-runner-17 --debug --ld-search-path /lib/x86_64-linux-gnu/ ./libsafety/libsafety_mutation.so -test-program=pytest -- -n8 --ignore-glob=misra/* \ No newline at end of file diff --git a/opendbc/safety/tests/test.sh b/opendbc/safety/tests/test.sh index e121721b..eae291f7 100755 --- a/opendbc/safety/tests/test.sh +++ b/opendbc/safety/tests/test.sh @@ -13,22 +13,16 @@ scons -j$(nproc) -D # run safety tests and generate coverage data pytest -n8 --ignore-glob=misra/* -if [ "$(uname)" = "Darwin" ]; then - GCOV_EXEC="/opt/homebrew/opt/llvm@17/bin/llvm-cov gcov" -else - GCOV_EXEC="llvm-cov-17 gcov" -fi - # generate and open report if [ "$1" == "--report" ]; then mkdir -p coverage-out - gcovr -r ../ --gcov-executable "$GCOV_EXEC" --html-nested coverage-out/index.html + gcovr -r ../ --html-nested coverage-out/index.html sensible-browser coverage-out/index.html fi # test coverage -GCOV="gcovr -r $DIR/../ --gcov-executable \"$GCOV_EXEC\" -d --fail-under-line=100 -e ^libsafety" -if ! GCOV_OUTPUT="$(eval $GCOV)"; then +GCOV="gcovr -r $DIR/../ -d --fail-under-line=100 -e ^libsafety" +if ! GCOV_OUTPUT="$($GCOV)"; then echo -e "FAILED:\n$GCOV_OUTPUT" exit 1 else