get mutation tests running on macOS (#2998)

* install mull on macOS

* runs

* run in ci

* more readable

* fail early

* who doesn't have curl

* fix that?

* uv sourcing

* unset that

* mit

* cleanup

.github/workflows/tests.yml

@@ -43,8 +43,14 @@ jobs:
 
   mutation:
     name: Safety mutation tests
-    runs-on: ${{ github.repository == 'commaai/opendbc' && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
+    runs-on: ${{ matrix.os }}
     timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ${{ github.repository == 'commaai/opendbc' && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
+          - os: ${{ github.repository == 'commaai/opendbc' && 'namespace-profile-macos-8x14' || 'macos-latest' }}
     env:
       GIT_REF: ${{ github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && github.event.before || format('origin/{0}', github.event.repository.default_branch) }}
     steps:

.gitignore

@@ -32,3 +32,4 @@ cppcheck-addon-ctu-file-list
 opendbc/safety/tests/coverage-out
 
 compile_commands.json
+.mull/

opendbc/safety/tests/libsafety/SConscript

@@ -47,11 +47,16 @@ if GetOption('ubsan'):
 
 # mutation env
 menv = env.Clone()
+if system == "Darwin":
+  menv.PrependENVPath('PATH', '/opt/homebrew/opt/llvm@17/bin')
+  mull_plugin = Dir('#').abspath + '/.mull/lib/mull-ir-frontend-17'
+else:
+  mull_plugin = '/usr/lib/mull-ir-frontend-17'
 menv['CC'] = 'clang-17'
 flags = [
   '-fprofile-instr-generate',
   '-fcoverage-mapping',
-  '-fpass-plugin=/usr/lib/mull-ir-frontend-17',
+  f'-fpass-plugin={mull_plugin}',
   '-g',
   '-grecord-command-line',
 ]
@@ -61,9 +66,6 @@ menv['CFLAGS'].remove('-fprofile-arcs')
 menv['CFLAGS'].remove('-ftest-coverage')
 
 for build_env, suffix in ((env, ""), (menv, "_mutation")):
-  # TODO: add macOS support
-  if (build_env == menv) and (system == "Darwin"):
-    continue
   safety = build_env.SharedObject(f"safety{suffix}.os", "safety.c")
   libsafety = build_env.SharedLibrary(f"libsafety{suffix}.so", [safety])
 

opendbc/safety/tests/mutation.sh

@@ -8,10 +8,14 @@ source $DIR/../../../setup.sh
 
 GIT_REF="${GIT_REF:-origin/master}"
 GIT_ROOT=$(git rev-parse --show-toplevel)
-MULL_OPS="mutators: [cxx_increment, cxx_decrement, cxx_comparison, cxx_boundary, cxx_bitwise_assignment, cxx_bitwise, cxx_arithmetic_assignment, cxx_arithmetic, cxx_remove_negation]"
-echo -e "$MULL_OPS" > $GIT_ROOT/mull.yml
-scons -j$(nproc) -D
-echo -e "timeout: 1000000\ngitDiffRef: $GIT_REF\ngitProjectRoot: $GIT_ROOT" >> $GIT_ROOT/mull.yml
+cat > $GIT_ROOT/mull.yml <<EOF
+mutators: [cxx_increment, cxx_decrement, cxx_comparison, cxx_boundary, cxx_bitwise_assignment, cxx_bitwise, cxx_arithmetic_assignment, cxx_arithmetic, cxx_remove_negation]
+timeout: 1000000
+gitDiffRef: $GIT_REF
+gitProjectRoot: $GIT_ROOT
+EOF
+
+scons -j4 -D
 
 export MUTATION=1
-mull-runner-17 --ld-search-path /lib/x86_64-linux-gnu/ ./libsafety/libsafety_mutation.so -test-program=pytest -- -n8 --ignore-glob=misra/*
+mull-runner-17 --debug --ld-search-path /lib/x86_64-linux-gnu/ ./libsafety/libsafety_mutation.so -test-program=pytest -- -n8 --ignore-glob=misra/*

setup.sh

@@ -8,11 +8,24 @@ export PYTHONPATH=$BASEDIR
 
 # *** dependencies install ***
 if [ "$(uname -s)" = "Linux" ]; then
-  # TODO: add macOS support
   if ! command -v "mull-runner-17" > /dev/null 2>&1; then
     curl -1sLf 'https://dl.cloudsmith.io/public/mull-project/mull-stable/setup.deb.sh' | sudo -E bash
     sudo apt-get update && sudo apt-get install -y clang-17 mull-17
   fi
+elif [ "$(uname -s)" = "Darwin" ]; then
+  if ! brew list llvm@17 &>/dev/null; then
+    brew install llvm@17
+  fi
+  if [ ! -f "$BASEDIR/.mull/bin/mull-runner-17" ]; then
+    MULL_VERSION="0.26.1"
+    MULL_ZIP="Mull-17-${MULL_VERSION}-LLVM-17.0-macOS-arm64-14.7.4.zip"
+    MULL_DIR="Mull-17-${MULL_VERSION}-LLVM-17.0-macOS-arm64-14.7.4"
+    curl -LO "https://github.com/mull-project/mull/releases/download/${MULL_VERSION}/${MULL_ZIP}"
+    unzip -o "$MULL_ZIP"
+    mv "$MULL_DIR" "$BASEDIR/.mull"
+    rm "$MULL_ZIP"
+  fi
+  export PATH="$BASEDIR/.mull/bin:$PATH"
 fi
 
 if ! command -v uv &>/dev/null; then