From 78b49ab155f6f6cb819df16f0327c5768ed6bb66 Mon Sep 17 00:00:00 2001
From: Maxime Desroches <desroches.maxime@gmail.com>
Date: Wed, 25 Sep 2024 21:56:09 -0700
Subject: [PATCH] safety mutation tests (#2040)

* mutation

* clone

* origin

* get diff

* better example

* better

* fix docker

* work on push

* make test fail

* real change

* test all modes

* ignore

* better

* reco

* fix

* no libpanda on device

* curl

* nl

* nl again

* delete

* clean

* clean

* this

* add this back

* cleanup
---
 .github/workflows/test.yaml  | 14 ++++++++++++++
 .gitignore                   |  1 +
 Dockerfile                   |  4 ++++
 SConstruct                   |  4 ++++
 tests/libpanda/SConscript    |  9 +++++++++
 tests/safety/install_mull.sh | 11 +++++++++++
 tests/safety/mutation.sh     | 20 ++++++++++++++++++++
 7 files changed, 63 insertions(+)
 create mode 100755 tests/safety/install_mull.sh
 create mode 100755 tests/safety/mutation.sh

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
index 5b62bf18f..12d956eec 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yaml
@@ -103,6 +103,20 @@ jobs:
         timeout-minutes: 5
         run: ${{ env.RUN }} "cd tests/misra && pytest -n8 test_mutation.py"
 
+  mutation:
+    name: Mutation tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # need master to get diff
+      - name: Build Docker image
+        run: eval "$BUILD"
+      - name: Mutation tests
+        timeout-minutes: 5
+        run: ${{ env.RUN }} "GIT_REF=${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.event.before || 'origin/master' }} cd tests/safety && ./mutation.sh"
+
   static_analysis:
     name: static analysis
     runs-on: ubuntu-latest
diff --git a/.gitignore b/.gitignore
index 8f7cb049b..89eed6e4c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -32,3 +32,4 @@ tests/safety/coverage.info
 
 *.profraw
 *.profdata
+mull.yml
diff --git a/Dockerfile b/Dockerfile
index a6f94826d..670283dd4 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -22,6 +22,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     apt-get update && apt-get install -y clang-17 && \
     ln -s $(which clang-17) /usr/bin/clang
 
+RUN apt-get update && apt-get install -y curl && \
+    curl -1sLf 'https://dl.cloudsmith.io/public/mull-project/mull-stable/setup.deb.sh' | bash && \
+    apt-get update && apt-get install -y mull-17
+
 ENV CPPCHECK_DIR=/tmp/cppcheck
 COPY tests/misra/install.sh /tmp/
 RUN /tmp/install.sh && rm -rf $CPPCHECK_DIR/.git/
diff --git a/SConstruct b/SConstruct
index 8a7609afe..6a97f4c17 100644
--- a/SConstruct
+++ b/SConstruct
@@ -12,6 +12,10 @@ AddOption('--compile_db',
           action='store_true',
           help='build clang compilation database')
 
+AddOption('--mutation',
+          action='store_true',
+          help='generate mutation-ready code')
+
 env = Environment(
   COMPILATIONDB_USE_ABSPATH=True,
   tools=["default", "compilation_db"],
diff --git a/tests/libpanda/SConscript b/tests/libpanda/SConscript
index 66dd563a7..fc33d85d3 100644
--- a/tests/libpanda/SConscript
+++ b/tests/libpanda/SConscript
@@ -28,6 +28,15 @@ else:
   env['CFLAGS'] += COVERAGE_FLAGS
   env['LINKFLAGS'] += COVERAGE_FLAGS
 
+if GetOption('mutation'):
+  flags = [
+    '-fpass-plugin=/usr/lib/mull-ir-frontend-17',
+    '-g',
+    '-grecord-command-line',
+  ]
+  env['CFLAGS'] += flags
+  env['LINKFLAGS'] += flags
+
 if GetOption('ubsan'):
   flags = [
     "-fsanitize=undefined",
diff --git a/tests/safety/install_mull.sh b/tests/safety/install_mull.sh
new file mode 100755
index 000000000..75b1042ec
--- /dev/null
+++ b/tests/safety/install_mull.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -e
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd $DIR
+
+if ! command -v "mull-runner-17" > /dev/null 2>&1; then
+  sudo apt-get update && sudo apt-get install -y curl clang-17
+  curl -1sLf 'https://dl.cloudsmith.io/public/mull-project/mull-stable/setup.deb.sh' | sudo -E bash
+  sudo apt-get update && sudo apt-get install -y mull-17
+fi
diff --git a/tests/safety/mutation.sh b/tests/safety/mutation.sh
new file mode 100755
index 000000000..23bba67c7
--- /dev/null
+++ b/tests/safety/mutation.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -e
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd $DIR
+
+$DIR/install_mull.sh
+
+scons --mutation -j$(nproc) -D
+
+GIT_REF="${GIT_REF:-origin/master}"
+echo -e "timeout: 10000\ngitDiffRef: $GIT_REF\ngitProjectRoot: ../../" > mull.yml
+
+SAFETY_MODELS=$(find * | grep "^test_.*\.py")
+for safety_model in ${SAFETY_MODELS[@]}; do
+  echo ""
+  echo ""
+  echo -e "Testing mutations on : $safety_model"
+  mull-runner-17 --ld-search-path /lib/x86_64-linux-gnu/ ../libpanda/libpanda.so -test-program=./$safety_model
+done