mac: fix setup and re-enable CI (#23365)

* rework mac_setup.sh * ignore casadi and onnx runtime in pipfile for Darwin * remove rust stuff * build and install casadi * add wget * explicit python3. dont do onnx just yet * resource compiler * replace acados Darwin lib w universal2 libs * relock Pipfile using linux machine * Update update_requirements.sh brackets * Update update_requirements.sh oops * ci: re-enable mac build * attempt to fix ci build: * pip stuff? * move rcc bin path to env script * oops * only mac * k im lazy. does ci like this?? * huh?? * * use curl * avoid casadi rebuilds * add comment to remove protobuf * host detect typo * python version * how did the builds not use swig? * better reinstall check * sometimes mac clang complain abt error limit * ci: build OP in venv * ci: bump to Big Sur * retry? idk why it failed @ acados link * * use macos-latest * move rcc path to scons * add extra paths for odd homebrew installs * acados source change * update macOS README * uh, maybe? * k nvm * ci: ok this is strange. might be scons bug? * fix conflicts: bump * just add cppcheck for pre-commit stuff * agane * cleanup * try that * fix path * no pyenv update on mac * source * fix rpath prefix? * no examples * fix exit * let's get this cached * add virtualenvs to cache * why did we cache that * let's see what's big * more * always ruyn * cache scons cache * better cache key * fix for partial hit * why so long Co-authored-by: Andrew Tec <andrest@trabus.com> Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com> Co-authored-by: Bruce Wayne <batman@openpilot-mac-mini.local> old-commit-hash: 66644217100f17e0a0c4f95f86aed2fd8c09c28c
2022-01-07 15:15:18 -08:00
parent 3c7e41d900
commit 192978753d
12 changed files with 129 additions and 81 deletions
--- a/.github/workflows/selfdrive_tests.yaml
+++ b/.github/workflows/selfdrive_tests.yaml
@@ -69,52 +69,62 @@ jobs:
                          rm -rf /tmp/scons_cache/* && \
                          scons -j$(nproc) --cache-populate"

-  #build_mac:
-  #  name: build macos
-  #  runs-on: macos-10.15
-  #  timeout-minutes: 60
-  #  steps:
-  #  - uses: actions/checkout@v2
-  #    with:
-  #      submodules: true
-  #  - name: Determine pre-existing Homebrew packages
-  #    if: steps.dependency-cache.outputs.cache-hit != 'true'
-  #    run: |
-  #      echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
-  #      ls -1 /usr/local/Cellar >> $GITHUB_ENV
-  #      echo 'EOF' >> $GITHUB_ENV
-  #  - name: Cache dependencies
-  #    id: dependency-cache
-  #    uses: actions/cache@v2
-  #    with:
-  #      path: |
-  #        ~/.pyenv
-  #        ~/Library/Caches/pip
-  #        ~/Library/Caches/pipenv
-  #        /usr/local/Cellar
-  #        ~/github_brew_cache_entries.txt
-  #      key: macos-cache-${{ hashFiles('tools/mac_setup.sh') }}
-  #  - name: Brew link restored dependencies
-  #    if: steps.dependency-cache.outputs.cache-hit == 'true'
-  #    run: |
-  #      while read pkg; do
-  #        brew link --force "$pkg" # `--force` for keg-only packages
-  #      done < ~/github_brew_cache_entries.txt
-  #  - name: Install dependencies
-  #    run: ./tools/mac_setup.sh
-  #  - name: Build openpilot
-  #    run: eval "$(pyenv init -)" && scons -j$(nproc)
-  #  - name: Remove pre-existing Homebrew packages for caching
-  #    if: steps.dependency-cache.outputs.cache-hit != 'true'
-  #    run: |
-  #      cd /usr/local/Cellar
-  #      new_cellar=$(ls -1)
-  #      comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do
-  #        if [[ $pkg != "zstd" ]]; then # caching step needs zstd
-  #          rm -rf "$pkg"
-  #        fi
-  #      done
-  #      comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt
+  build_mac:
+    name: build macos
+    runs-on: macos-latest
+    timeout-minutes: 60
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - name: Determine pre-existing Homebrew packages
+      if: steps.dependency-cache.outputs.cache-hit != 'true'
+      run: |
+        echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
+        ls -1 /usr/local/Cellar >> $GITHUB_ENV
+        echo 'EOF' >> $GITHUB_ENV
+    - name: Cache dependencies
+      id: dependency-cache
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/.pyenv
+          ~/.local/share/virtualenvs/
+          /usr/local/Cellar
+          ~/github_brew_cache_entries.txt
+          /tmp/scons_cache
+        key: macos-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'Pipfile*') }}
+        restore-keys: macos-
+    - name: Brew link restored dependencies
+      run: |
+        if [ -f ~/github_brew_cache_entries.txt ]; then
+          while read pkg; do
+            brew link --force "$pkg" # `--force` for keg-only packages
+          done < ~/github_brew_cache_entries.txt
+        else
+          echo "Cache entries not found"
+        fi
+    - name: Install dependencies
+      run: ./tools/mac_setup.sh
+    - name: Build openpilot
+      run: |
+        source tools/openpilot_env.sh
+        pipenv run selfdrive/manager/build.py
+
+        # cleanup scons cache
+        rm -rf /tmp/scons_cache/
+        pipenv run scons -j$(nproc) --cache-populate
+    - name: Remove pre-existing Homebrew packages for caching
+      if: steps.dependency-cache.outputs.cache-hit != 'true'
+      run: |
+        cd /usr/local/Cellar
+        new_cellar=$(ls -1)
+        comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do
+          if [[ $pkg != "zstd" ]]; then # caching step needs zstd
+            rm -rf "$pkg"
+          fi
+        done
+        comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt

  build_webcam:
    name: build webcam
--- a/BIN
+++ b/BIN
--- a/Pipfile.lock
+++ b/Pipfile.lock
--- a/9
+++ b/9
@@ -66,7 +66,7 @@ lenv = {
  "LD_LIBRARY_PATH": [Dir(f"#third_party/acados/{arch}/lib").abspath],
  "PYTHONPATH": Dir("#").abspath + ":" + Dir("#pyextra/").abspath,

-  "ACADOS_SOURCE_DIR": Dir("#third_party/acados/acados").abspath,
+  "ACADOS_SOURCE_DIR": Dir("#third_party/acados/include/acados").abspath,
  "ACADOS_PYTHON_INTERFACE_PATH": Dir("#pyextra/acados_template").abspath,
  "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer",
 }
@@ -125,8 +125,10 @@ else:
      f"#third_party/libyuv/{yuv_dir}/lib",
      "/usr/local/lib",
      "/opt/homebrew/lib",
+      "/usr/local/Homebrew/Library",
      "/usr/local/opt/openssl/lib",
      "/opt/homebrew/opt/openssl/lib",
+      "/usr/local/Cellar",
      f"#third_party/acados/{arch}/lib",
      "/System/Library/Frameworks/OpenGL.framework/Libraries",
    ]
@@ -134,6 +136,7 @@ else:
    cxxflags += ["-DGL_SILENCE_DEPRECATION"]
    cpppath += [
      "/opt/homebrew/include",
+      "/usr/local/include",
      "/usr/local/opt/openssl/include",
      "/opt/homebrew/opt/openssl/include"
    ]
@@ -235,6 +238,9 @@ env = Environment(
  tools=["default", "cython", "compilation_db"],
 )

+if arch == "Darwin":
+  env['RPATHPREFIX'] = "-rpath "
+
 if GetOption('compile_db'):
  env.CompilationDatabase('compile_commands.json')

@@ -299,6 +305,7 @@ if arch == "Darwin":
  qt_dirs += [f"{qt_env['QTDIR']}/include/Qt{m}" for m in qt_modules]
  qt_env["LINKFLAGS"] += ["-F" + os.path.join(qt_env['QTDIR'], "lib")]
  qt_env["FRAMEWORKS"] += [f"Qt{m}" for m in qt_modules] + ["OpenGL"]
+  qt_env.AppendENVPath('PATH', os.path.join(qt_env['QTDIR'], "bin"))
 elif arch == "aarch64":
  qt_env['QTDIR'] = "/system/comma/usr"
  qt_dirs = [
--- a/third_party/acados/Darwin/lib/libacados.dylib
+++ b/third_party/acados/Darwin/lib/libacados.dylib
--- a/third_party/acados/Darwin/lib/libblasfeo.dylib
+++ b/third_party/acados/Darwin/lib/libblasfeo.dylib
--- a/third_party/acados/Darwin/lib/libhpipm.dylib
+++ b/third_party/acados/Darwin/lib/libhpipm.dylib
--- a/third_party/acados/Darwin/lib/libqpOASES_e.3.1.dylib
+++ b/third_party/acados/Darwin/lib/libqpOASES_e.3.1.dylib
--- a/tools/mac_setup.sh
+++ b/tools/mac_setup.sh
@@ -1,4 +1,6 @@
-#!/bin/bash -e
+#!/bin/bash
+
+set -e

 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 ROOT="$(cd $DIR/../ && pwd)"
@@ -7,15 +9,16 @@ ROOT="$(cd $DIR/../ && pwd)"
 if [[ $(command -v brew) == "" ]]; then
  echo "Installing Hombrew"
  /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
+  echo "[ ] installed brew t=$SECONDS"
 fi

+# TODO: remove protobuf,protobuf-c,swig when casadi can be pip installed
 brew bundle --file=- <<-EOS
 brew "cmake"
+brew "cppcheck"
 brew "git-lfs"
 brew "zlib"
 brew "bzip2"
-brew "rust"
-brew "rustup-init"
 brew "capnp"
 brew "coreutils"
 brew "eigen"
@@ -29,21 +32,20 @@ brew "openssl"
 brew "pyenv"
 brew "qt@5"
 brew "zeromq"
+brew "protobuf"
+brew "protobuf-c"
+brew "swig"
 cask "gcc-arm-embedded"
 EOS

+echo "[ ] finished brew install t=$SECONDS"
+
 if [[ $SHELL == "/bin/zsh" ]]; then
  RC_FILE="$HOME/.zshrc"
 elif [[ $SHELL == "/bin/bash" ]]; then
  RC_FILE="$HOME/.bash_profile"
 fi

-# TODO: get rid of this somehow
-# Build requirements for macOS
-# https://github.com/pyenv/pyenv/issues/1740
-# https://github.com/pyca/cryptography/blob/main/docs/installation.rst
-rustup-init -y
-
 export LDFLAGS="$LDFLAGS -L/usr/local/opt/zlib/lib"
 export LDFLAGS="$LDFLAGS -L/usr/local/opt/bzip2/lib"
 export LDFLAGS="$LDFLAGS -L/usr/local/opt/openssl@1.1/lib"
@@ -55,17 +57,40 @@ export PATH="$PATH:/usr/local/bin"

 # openpilot environment
 if [ -z "$OPENPILOT_ENV" ] && [ -n "$RC_FILE" ] && [ -z "$CI" ]; then
-  echo "export PATH=\"\$PATH:$HOME/.cargo/bin\"" >> $RC_FILE
  echo "source $ROOT/tools/openpilot_env.sh" >> $RC_FILE
-  export PATH="$PATH:\"\$HOME/.cargo/bin\""
  source "$ROOT/tools/openpilot_env.sh"
  echo "Added openpilot_env to RC file: $RC_FILE"
 fi

 # install python dependencies
 $ROOT/update_requirements.sh
+eval "$(pyenv init --path)"
+echo "[ ] installed python dependencies t=$SECONDS"
+
+# install casadi
+VENV=`pipenv --venv`
+PYTHON_VER=3.8
+PYTHON_VERSION=$(cat $ROOT/.python-version)
+if [ ! -f "$VENV/include/casadi/casadi.hpp" ]; then
+  echo "-- casadi manual install"
+  cd /tmp/ && curl -L https://github.com/casadi/casadi/archive/refs/tags/ge6.tar.gz --output casadi.tar.gz
+  tar -xzf casadi.tar.gz
+  cd casadi-ge6/ && mkdir -p build && cd build
+  cmake .. \
+    -DWITH_PYTHON=ON \
+    -DWITH_EXAMPLES=OFF \
+    -DCMAKE_INSTALL_PREFIX:PATH=$VENV \
+    -DPYTHON_PREFIX:PATH=$VENV/lib/python$PYTHON_VER/site-packages \
+    -DPYTHON_LIBRARY:FILEPATH=$HOME/.pyenv/versions/$PYTHON_VERSION/lib/libpython$PYTHON_VER.dylib \
+    -DPYTHON_EXECUTABLE:FILEPATH=$HOME/.pyenv/versions/$PYTHON_VERSION/bin/python \
+    -DPYTHON_INCLUDE_DIR:PATH=$HOME/.pyenv/versions/$PYTHON_VERSION/include/python$PYTHON_VER \
+    -DCMAKE_CXX_FLAGS="-ferror-limit=0" -DCMAKE_C_FLAGS="-ferror-limit=0"
+  CFLAGS="-ferror-limit=0" make -j$(nproc) && make install
+else
+  echo "----   casadi found in venv. skipping build   ----"
+fi

 echo
-echo "----   FINISH OPENPILOT SETUP   ----"
-echo "Configure your active shell env by running:"
+echo "----   OPENPILOT SETUP DONE   ----"
+echo "Open a new shell or configure your active shell env by running:"
 echo "source $RC_FILE"
--- a/tools/openpilot_env.sh
+++ b/tools/openpilot_env.sh
@@ -9,13 +9,13 @@ if [ -z "$OPENPILOT_ENV" ]; then
  export PYENV_ROOT="$HOME/.pyenv"

  if [[ "$(uname)" == 'Linux' ]]; then
-    eval "$(pyenv init --path)"
    eval "$(pyenv virtualenv-init -)"
  elif [[ "$(uname)" == 'Darwin' ]]; then
    # msgq doesn't work on mac
    export ZMQ=1
    export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
  fi
+  eval "$(pyenv init --path)"
  eval "$(pyenv init -)"

  export OPENPILOT_ENV=1
--- a/tools/ubuntu_setup.sh
+++ b/tools/ubuntu_setup.sh
@@ -134,6 +134,6 @@ if [ -z "$OPENPILOT_ENV" ]; then
 fi

 echo
-echo "----   FINISH OPENPILOT SETUP   ----"
-echo "Configure your active shell env by running:"
+echo "----   OPENPILOT SETUP DONE   ----"
+echo "Open a new shell or configure your active shell env by running:"
 echo "source ~/.bashrc"
--- a/update_requirements.sh
+++ b/update_requirements.sh
@@ -1,4 +1,6 @@
-#!/bin/bash -e
+#!/bin/bash
+
+set -e

 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 cd $DIR
@@ -13,15 +15,18 @@ export MAKEFLAGS="-j$(nproc)"

 PYENV_PYTHON_VERSION=$(cat .python-version)
 if ! pyenv prefix ${PYENV_PYTHON_VERSION} &> /dev/null; then
-  echo "pyenv update ..."
-  pyenv update
+  # no pyenv update on mac
+  if [ "$(uname)" == "Linux" ]; then
+    echo "pyenv update ..."
+    pyenv update
+  fi
  echo "python ${PYENV_PYTHON_VERSION} install ..."
  CONFIGURE_OPTS="--enable-shared" pyenv install -f ${PYENV_PYTHON_VERSION}
 fi
+eval "$(pyenv init --path)"

-echo "pip install ..."
+echo "update pip"
 pip install pip==21.3.1
-echo "pipenv install ..."
 pip install pipenv==2021.11.23

 if [ -d "./xx" ]; then
@@ -36,7 +41,7 @@ else
  RUN=""
 fi

-echo "pip packages install ..."
+echo "pip packages install..."
 pipenv install --dev --deploy --clear
 pyenv rehash

@@ -45,4 +50,5 @@ if [ -f "$DIR/.pre-commit-config.yaml" ]; then
  $RUN pre-commit install
  [ -d "./xx" ] && (cd xx && $RUN pre-commit install)
  [ -d "./notebooks" ] && (cd notebooks && $RUN pre-commit install)
+  echo "pre-commit hooks installed"
 fi