safety mutation tests (#2040)

* mutation

* clone

* origin

* get diff

* better example

* better

* fix docker

* work on push

* make test fail

* real change

* test all modes

* ignore

* better

* reco

* fix

* no libpanda on device

* curl

* nl

* nl again

* delete

* clean

* clean

* this

* add this back

* cleanup

.github/workflows/test.yaml

@@ -103,6 +103,20 @@ jobs:
         timeout-minutes: 5
         run: ${{ env.RUN }} "cd tests/misra && pytest -n8 test_mutation.py"
 
+  mutation:
+    name: Mutation tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # need master to get diff
+      - name: Build Docker image
+        run: eval "$BUILD"
+      - name: Mutation tests
+        timeout-minutes: 5
+        run: ${{ env.RUN }} "GIT_REF=${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.event.before || 'origin/master' }} cd tests/safety && ./mutation.sh"
+
   static_analysis:
     name: static analysis
     runs-on: ubuntu-latest

.gitignore

@@ -32,3 +32,4 @@ tests/safety/coverage.info
 
 *.profraw
 *.profdata
+mull.yml

Dockerfile

@@ -22,6 +22,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     apt-get update && apt-get install -y clang-17 && \
     ln -s $(which clang-17) /usr/bin/clang
 
+RUN apt-get update && apt-get install -y curl && \
+    curl -1sLf 'https://dl.cloudsmith.io/public/mull-project/mull-stable/setup.deb.sh' | bash && \
+    apt-get update && apt-get install -y mull-17
+
 ENV CPPCHECK_DIR=/tmp/cppcheck
 COPY tests/misra/install.sh /tmp/
 RUN /tmp/install.sh && rm -rf $CPPCHECK_DIR/.git/

SConstruct

@@ -12,6 +12,10 @@ AddOption('--compile_db',
           action='store_true',
           help='build clang compilation database')
 
+AddOption('--mutation',
+          action='store_true',
+          help='generate mutation-ready code')
+
 env = Environment(
   COMPILATIONDB_USE_ABSPATH=True,
   tools=["default", "compilation_db"],

tests/libpanda/SConscript

@@ -28,6 +28,15 @@ else:
   env['CFLAGS'] += COVERAGE_FLAGS
   env['LINKFLAGS'] += COVERAGE_FLAGS
 
+if GetOption('mutation'):
+  flags = [
+    '-fpass-plugin=/usr/lib/mull-ir-frontend-17',
+    '-g',
+    '-grecord-command-line',
+  ]
+  env['CFLAGS'] += flags
+  env['LINKFLAGS'] += flags
+
 if GetOption('ubsan'):
   flags = [
     "-fsanitize=undefined",

tests/safety/install_mull.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -e
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd $DIR
+
+if ! command -v "mull-runner-17" > /dev/null 2>&1; then
+  sudo apt-get update && sudo apt-get install -y curl clang-17
+  curl -1sLf 'https://dl.cloudsmith.io/public/mull-project/mull-stable/setup.deb.sh' | sudo -E bash
+  sudo apt-get update && sudo apt-get install -y mull-17
+fi

tests/safety/mutation.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -e
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd $DIR
+
+$DIR/install_mull.sh
+
+scons --mutation -j$(nproc) -D
+
+GIT_REF="${GIT_REF:-origin/master}"
+echo -e "timeout: 10000\ngitDiffRef: $GIT_REF\ngitProjectRoot: ../../" > mull.yml
+
+SAFETY_MODELS=$(find * | grep "^test_.*\.py")
+for safety_model in ${SAFETY_MODELS[@]}; do
+  echo ""
+  echo ""
+  echo -e "Testing mutations on : $safety_model"
+  mull-runner-17 --ld-search-path /lib/x86_64-linux-gnu/ ../libpanda/libpanda.so -test-program=./$safety_model
+done