Merge remote-tracking branch 'commaai/panda/master' into camera-scc-long

.github/workflows/test.yaml

@@ -103,6 +103,20 @@ jobs:
         timeout-minutes: 5
         run: ${{ env.RUN }} "cd tests/misra && pytest -n8 test_mutation.py"
 
+  mutation:
+    name: Mutation tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # need master to get diff
+      - name: Build Docker image
+        run: eval "$BUILD"
+      - name: Mutation tests
+        timeout-minutes: 5
+        run: ${{ env.RUN }} "GIT_REF=${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.event.before || 'origin/master' }} cd tests/safety && ./mutation.sh"
+
   static_analysis:
     name: static analysis
     runs-on: ubuntu-latest

.gitignore

@@ -29,3 +29,7 @@ nosetests.xml
 *.gcno
 tests/safety/coverage-out
 tests/safety/coverage.info
+
+*.profraw
+*.profdata
+mull.yml

Dockerfile

@@ -18,7 +18,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     apt clean && \
     cd /usr/lib/gcc/arm-none-eabi/* && \
     rm -rf arm/ && \
-    rm -rf thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
+    rm -rf thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp && \
+    apt-get update && apt-get install -y clang-17 && \
+    ln -s $(which clang-17) /usr/bin/clang
+
+RUN apt-get update && apt-get install -y curl && \
+    curl -1sLf 'https://dl.cloudsmith.io/public/mull-project/mull-stable/setup.deb.sh' | bash && \
+    apt-get update && apt-get install -y mull-17
 
 ENV CPPCHECK_DIR=/tmp/cppcheck
 COPY tests/misra/install.sh /tmp/

README.md

@@ -49,7 +49,7 @@ In addition, we run the [ruff linter](https://github.com/astral-sh/ruff) and [my
 Setup dependencies:
 ```bash
 # Ubuntu
-sudo apt-get install dfu-util gcc-arm-none-eabi python3-pip libffi-dev git
+sudo apt-get install dfu-util gcc-arm-none-eabi python3-pip libffi-dev git clang-17
 
 # macOS
 brew install --cask gcc-arm-embedded

SConstruct

@@ -16,6 +16,10 @@ AddOption('--compile_db',
           action='store_true',
           help='build clang compilation database')
 
+AddOption('--mutation',
+          action='store_true',
+          help='generate mutation-ready code')
+
 env = Environment(
   COMPILATIONDB_USE_ABSPATH=True,
   tools=["default", "compilation_db"],

tests/libpanda/SConscript

@@ -21,6 +21,18 @@ env = Environment(
 if system == "Darwin":
   env.PrependENVPath('PATH', '/opt/homebrew/bin')
 
+if GetOption('mutation'):
+  env['CC'] = 'clang-17'
+  flags = [
+    '-fprofile-instr-generate',
+    '-fcoverage-mapping',
+    '-fpass-plugin=/usr/lib/mull-ir-frontend-17',
+    '-g',
+    '-grecord-command-line',
+  ]
+  env['CFLAGS'] += flags
+  env['LINKFLAGS'] += flags
+
 if GetOption('ubsan'):
   flags = [
     "-fsanitize=undefined",

tests/safety/install_mull.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -e
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd $DIR
+
+if ! command -v "mull-runner-17" > /dev/null 2>&1; then
+  sudo apt-get update && sudo apt-get install -y curl clang-17
+  curl -1sLf 'https://dl.cloudsmith.io/public/mull-project/mull-stable/setup.deb.sh' | sudo -E bash
+  sudo apt-get update && sudo apt-get install -y mull-17
+fi

tests/safety/mutation.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -e
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd $DIR
+
+$DIR/install_mull.sh
+
+GIT_REF="${GIT_REF:-origin/master}"
+GIT_ROOT=$(git rev-parse --show-toplevel)
+echo -e "mutators:\n  - cxx_all" > $GIT_ROOT/mull.yml
+scons --mutation -j$(nproc) -D
+echo -e "timeout: 10000\ngitDiffRef: $GIT_REF\ngitProjectRoot: $GIT_ROOT" >> $GIT_ROOT/mull.yml
+
+SAFETY_MODELS=$(find * | grep "^test_.*\.py")
+for safety_model in ${SAFETY_MODELS[@]}; do
+  echo ""
+  echo ""
+  echo -e "Testing mutations on : $safety_model"
+  mull-runner-17 --ld-search-path /lib/x86_64-linux-gnu/ ../libpanda/libpanda.so -test-program=./$safety_model
+done