diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5b62bf18f..12d956eec 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -103,6 +103,20 @@ jobs: timeout-minutes: 5 run: ${{ env.RUN }} "cd tests/misra && pytest -n8 test_mutation.py" + mutation: + name: Mutation tests + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # need master to get diff + - name: Build Docker image + run: eval "$BUILD" + - name: Mutation tests + timeout-minutes: 5 + run: ${{ env.RUN }} "GIT_REF=${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.event.before || 'origin/master' }} cd tests/safety && ./mutation.sh" + static_analysis: name: static analysis runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 8f7cb049b..89eed6e4c 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ tests/safety/coverage.info *.profraw *.profdata +mull.yml diff --git a/Dockerfile b/Dockerfile index a6f94826d..670283dd4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,6 +22,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ apt-get update && apt-get install -y clang-17 && \ ln -s $(which clang-17) /usr/bin/clang +RUN apt-get update && apt-get install -y curl && \ + curl -1sLf 'https://dl.cloudsmith.io/public/mull-project/mull-stable/setup.deb.sh' | bash && \ + apt-get update && apt-get install -y mull-17 + ENV CPPCHECK_DIR=/tmp/cppcheck COPY tests/misra/install.sh /tmp/ RUN /tmp/install.sh && rm -rf $CPPCHECK_DIR/.git/ diff --git a/SConstruct b/SConstruct index 8a7609afe..6a97f4c17 100644 --- a/SConstruct +++ b/SConstruct @@ -12,6 +12,10 @@ AddOption('--compile_db', action='store_true', help='build clang compilation database') +AddOption('--mutation', + action='store_true', + help='generate mutation-ready code') + env = Environment( COMPILATIONDB_USE_ABSPATH=True, tools=["default", "compilation_db"], diff --git a/tests/libpanda/SConscript b/tests/libpanda/SConscript index 66dd563a7..fc33d85d3 100644 --- a/tests/libpanda/SConscript +++ b/tests/libpanda/SConscript @@ -28,6 +28,15 @@ else: env['CFLAGS'] += COVERAGE_FLAGS env['LINKFLAGS'] += COVERAGE_FLAGS +if GetOption('mutation'): + flags = [ + '-fpass-plugin=/usr/lib/mull-ir-frontend-17', + '-g', + '-grecord-command-line', + ] + env['CFLAGS'] += flags + env['LINKFLAGS'] += flags + if GetOption('ubsan'): flags = [ "-fsanitize=undefined", diff --git a/tests/safety/install_mull.sh b/tests/safety/install_mull.sh new file mode 100755 index 000000000..75b1042ec --- /dev/null +++ b/tests/safety/install_mull.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -e + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +cd $DIR + +if ! command -v "mull-runner-17" > /dev/null 2>&1; then + sudo apt-get update && sudo apt-get install -y curl clang-17 + curl -1sLf 'https://dl.cloudsmith.io/public/mull-project/mull-stable/setup.deb.sh' | sudo -E bash + sudo apt-get update && sudo apt-get install -y mull-17 +fi diff --git a/tests/safety/mutation.sh b/tests/safety/mutation.sh new file mode 100755 index 000000000..23bba67c7 --- /dev/null +++ b/tests/safety/mutation.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -e + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +cd $DIR + +$DIR/install_mull.sh + +scons --mutation -j$(nproc) -D + +GIT_REF="${GIT_REF:-origin/master}" +echo -e "timeout: 10000\ngitDiffRef: $GIT_REF\ngitProjectRoot: ../../" > mull.yml + +SAFETY_MODELS=$(find * | grep "^test_.*\.py") +for safety_model in ${SAFETY_MODELS[@]}; do + echo "" + echo "" + echo -e "Testing mutations on : $safety_model" + mull-runner-17 --ld-search-path /lib/x86_64-linux-gnu/ ../libpanda/libpanda.so -test-program=./$safety_model +done