macOS: fix build issues & re-enable CI (#28258)

* Uncomment build_mac job * Add PYCURL_CURL_CONFIG * Acados universal binary * Remove x86 macos references from sconscript * Add missing include in cabana * Update cereal * Update workflow * Remove unnecessary libpaths for darwin * Add missing path to cached dirs * Fix path in cache preparation step * Add poetry caches to CI cache * Compile acados with lower deployment target for CI * Update cereal * Pass -rpath in linker flags as scons doed not support RPATH on macos * Use scons api instead of os * @rpath in install name of acados dylibs * SConstruct cleanup * fix liblocationd tests by adding lib suffix based on platform * Update cereal * Update opendbc * Update opendbc * Add acados gitignore to release files * Update cereal * Update cereal * Add scons_cache to build cache * Add Caskroom to cache * Fix typo * Link all packages at once, instead of one by one * Run cleanup stage using poetry * Remove casks from cache * Move scons cache to separate cache step * Save scons cache only on master * Remove restore-keys from save-scons-cache step * Uncomment if conditions for scons save * Add gcc-arm-embedded cask cache to cache * Custom handling of gcc-arm-embedded toolchain cache * Rename dep cache key * Exclude .fseventsd from cache * Fix glob pattern * Remove .feventsd before caching * Run mac_setup only if dependency cache-hit != true * Update cereal to master * Add overwrite flag to brew link * Remove manual casadi build from mac_setup * Remove restore-keys from dependency cache * Remove linux requirement for casadi * Restore restore-keys to dependency cache
2023-06-21 22:14:20 +02:00 · 2023-06-21 22:14:20 +02:00 · 45a6834a74
parent e3233cdaf2
commit 45a6834a74
15 changed files with 120 additions and 112 deletions
--- a/.github/workflows/selfdrive_tests.yaml
+++ b/.github/workflows/selfdrive_tests.yaml
@ -81,61 +81,87 @@ jobs:
          ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \
                          scons -j$(nproc) --cache-populate"

-  #build_mac:
-  #  name: build macos
-  #  runs-on: macos-latest
-  #  steps:
-  #  - uses: actions/checkout@v3
-  #    with:
-  #      submodules: true
-  #  - name: Determine pre-existing Homebrew packages
-  #    if: steps.dependency-cache.outputs.cache-hit != 'true'
-  #    run: |
-  #      echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
-  #      ls -1 /usr/local/Cellar >> $GITHUB_ENV
-  #      echo 'EOF' >> $GITHUB_ENV
-  #  - name: Cache dependencies
-  #    id: dependency-cache
-  #    uses: actions/cache@v2
-  #    with:
-  #      path: |
-  #        ~/.pyenv
-  #        ~/.local/share/virtualenvs/
-  #        /usr/local/Cellar
-  #        ~/github_brew_cache_entries.txt
-  #        /tmp/scons_cache
-  #      key: macos-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'poetry.lock') }}
-  #      restore-keys: macos-
-  #  - name: Brew link restored dependencies
-  #    run: |
-  #      if [ -f ~/github_brew_cache_entries.txt ]; then
-  #        while read pkg; do
-  #          brew link --force "$pkg" # `--force` for keg-only packages
-  #        done < ~/github_brew_cache_entries.txt
-  #      else
-  #        echo "Cache entries not found"
-  #      fi
-  #  - name: Install dependencies
-  #    run: ./tools/mac_setup.sh
-  #  - name: Build openpilot
-  #    run: |
-  #      source tools/openpilot_env.sh
-  #      poetry run selfdrive/manager/build.py
-  #
-  #      # cleanup scons cache
-  #      rm -rf /tmp/scons_cache/
-  #      poetry run scons -j$(nproc) --cache-populate
-  #  - name: Remove pre-existing Homebrew packages for caching
-  #    if: steps.dependency-cache.outputs.cache-hit != 'true'
-  #    run: |
-  #      cd /usr/local/Cellar
-  #      new_cellar=$(ls -1)
-  #      comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do
-  #        if [[ $pkg != "zstd" ]]; then # caching step needs zstd
-  #          rm -rf "$pkg"
-  #        fi
-  #      done
-  #      comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt
+  build_mac:
+    name: build macos
+    runs-on: macos-latest
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+    - name: Determine pre-existing Homebrew packages
+      if: steps.dependency-cache.outputs.cache-hit != 'true'
+      run: |
+        echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
+        brew list --formula -1 >> $GITHUB_ENV
+        echo 'EOF' >> $GITHUB_ENV
+    - name: Restore scons cache
+      id: scons-restore-cache
+      uses: actions/cache/restore@v3
+      with:
+        path: /tmp/scons_cache
+        key: macos_scons-${{ github.sha }}
+        restore-keys: macos_scons-
+    - name: Cache dependencies
+      id: dependency-cache
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/github_brew_cache_entries.txt
+          ~/.pyenv
+          ~/Library/Caches/pypoetry
+          /usr/local/Cellar
+          /usr/local/opt
+          /usr/local/Caskroom/gcc-arm-*
+          /opt/homebrew/Cellar
+          /opt/homebrew/opt
+          /opt/homebrew/Caskroom/gcc-arm-*
+          /Applications/ArmGNUToolchain/*/*/*
+        key: macos_deps-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'poetry.lock') }}
+        restore-keys: macos_deps-
+    - name: Brew link restored dependencies
+      run: |
+        if [ -f ~/github_brew_cache_entries.txt ]; then
+          brew link --force --overwrite $(cat ~/github_brew_cache_entries.txt) # `--force` for keg-only packages
+          if [ -d /Applications/ArmGNUToolchain ]; then # link gcc-arm-embedded manually
+            GCC_TOOLCHAIN="$(echo /Applications/ArmGNUToolchain/**/**/bin)"
+            echo "$GCC_TOOLCHAIN" >> $GITHUB_PATH
+          fi
+        else
+          echo "Cache entries not found"
+        fi
+    - name: Install dependencies
+      if: steps.dependency-cache.outputs.cache-hit != 'true'
+      run: ./tools/mac_setup.sh
+    - name: Build openpilot
+      run: |
+        source tools/openpilot_env.sh
+        poetry run scons -j$(nproc)
+    - name: Pre Cache - Cleanup scons cache
+      if: github.ref == 'refs/heads/master'
+      run: |
+        source tools/openpilot_env.sh
+        rm -rf /tmp/scons_cache/*
+        poetry run scons -j$(nproc) --cache-populate
+    - name: Pre Cache - Remove pre-existing Homebrew packages
+      if: steps.dependency-cache.outputs.cache-hit != 'true'
+      run: |
+        new_cellar=$(brew list --formula -1)
+        comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do
+          if [[ $pkg != "zstd" ]]; then # caching step needs zstd
+            rm -rf "$(brew --cellar)/$pkg"
+          fi
+        done
+        comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt
+        # .fseventsd directory causes permission errors in dep caching step
+        # its used by the system to observe changes within the directory - toolchain works without it, just remove it
+        sudo rm -rf /Applications/ArmGNUToolchain/*/*/.fseventsd
+    - name: Save scons cache
+      id: scons-save-cache
+      uses: actions/cache/save@v3
+      if: github.ref == 'refs/heads/master'
+      with:
+        path: /tmp/scons_cache
+        key: macos_scons-${{ github.sha }}

  docker_push:
    name: docker push
--- a/43
+++ b/43
@ -67,17 +67,22 @@ AddOption('--no-test',
 real_arch = arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
 if platform.system() == "Darwin":
  arch = "Darwin"
+  brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip()

 if arch == "aarch64" and AGNOS:
  arch = "larch64"

+# create symlink to lib dir of current arch, so the ACADOS_SOURCE_DIR would have valid structure
+acados_lib_path = Dir(f"#third_party/acados/lib")
+if not Dir(f"#third_party/acados/lib").exists():
+  os.symlink(Dir(f"#third_party/acados/{arch}/lib").abspath, acados_lib_path.abspath)

 lenv = {
  "PATH": os.environ['PATH'],
  "LD_LIBRARY_PATH": [Dir(f"#third_party/acados/{arch}/lib").abspath],
  "PYTHONPATH": Dir("#").abspath,

-  "ACADOS_SOURCE_DIR": Dir("#third_party/acados/include/acados").abspath,
+  "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
  "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
  "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
 }
@ -110,26 +115,21 @@ else:
  cflags = []
  cxxflags = []
  cpppath = []
+  rpath += [
+    Dir("#cereal").abspath,
+    Dir("#common").abspath
+  ]

  # MacOS
  if arch == "Darwin":
-    if real_arch == "x86_64":
-      lenv["TERA_PATH"] = Dir("#").abspath + f"/third_party/acados/Darwin_x86_64/t_renderer"
-
-    brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip()
    yuv_dir = "mac" if real_arch != "arm64" else "mac_arm64"
    libpath = [
      f"#third_party/libyuv/{yuv_dir}/lib",
+      f"#third_party/acados/{arch}/lib",
      f"{brew_prefix}/lib",
-      f"{brew_prefix}/Library",
      f"{brew_prefix}/opt/openssl@3.0/lib",
-      f"{brew_prefix}/Cellar",
      "/System/Library/Frameworks/OpenGL.framework/Libraries",
    ]
-    if real_arch == "x86_64":
-      libpath.append(f"#third_party/acados/Darwin_x86_64/lib")
-    else:
-      libpath.append(f"#third_party/acados/{arch}/lib")

    cflags += ["-DGL_SILENCE_DEPRECATION"]
    cxxflags += ["-DGL_SILENCE_DEPRECATION"]
@ -137,6 +137,7 @@ else:
      f"{brew_prefix}/include",
      f"{brew_prefix}/opt/openssl@3.0/include",
    ]
+    lenv["DYLD_LIBRARY_PATH"] = lenv["LD_LIBRARY_PATH"]
  # Linux 86_64
  else:
    libpath = [
@ -149,12 +150,9 @@ else:
      "/usr/lib",
      "/usr/local/lib",
    ]
-
-  rpath += [
-    Dir("#third_party/snpe/x86_64-linux-clang").abspath,
-    Dir("#cereal").abspath,
-    Dir("#common").abspath
-  ]
+    rpath += [
+      Dir("#third_party/snpe/x86_64-linux-clang").abspath,
+    ]

 if GetOption('asan'):
  ccflags = ["-fsanitize=address", "-fno-omit-frame-pointer"]
@ -231,7 +229,9 @@ env = Environment(
 )

 if arch == "Darwin":
-  env['RPATHPREFIX'] = "-rpath "
+  # RPATH is not supported on macOS, instead use the linker flags
+  darwin_rpath_link_flags = [f"-Wl,-rpath,{path}" for path in env["RPATH"]]
+  env["LINKFLAGS"] += darwin_rpath_link_flags

 if GetOption('compile_db'):
  env.CompilationDatabase('compile_commands.json')
@ -271,7 +271,7 @@ envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-decla

 envCython["LIBS"] = []
 if arch == "Darwin":
-  envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"]
+  envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"] + darwin_rpath_link_flags
 elif arch == "aarch64":
  envCython["LINKFLAGS"] = ["-shared"]
  envCython["LIBS"] = [os.path.basename(py_include)]
@ -286,10 +286,7 @@ qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "Multimedia", "

 qt_libs = []
 if arch == "Darwin":
-  if real_arch == "arm64":
-    qt_env['QTDIR'] = "/opt/homebrew/opt/qt@5"
-  else:
-    qt_env['QTDIR'] = "/usr/local/opt/qt@5"
+  qt_env['QTDIR'] = f"{brew_prefix}/opt/qt@5"
  qt_dirs = [
    os.path.join(qt_env['QTDIR'], "include"),
  ]
--- a/poetry.lock
+++ b/poetry.lock
@ -4815,7 +4815,7 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"]
 [metadata]
 lock-version = "1.1"
 python-versions = "~3.8"
-content-hash = "15f51cce43698998497c179525f7a804f6e16b668d8d8b531aae07b0b0ee79d6"
+content-hash = "08d4fdc2579454f8df4b7edc5b537e8b1e3216506ac5baf1f80ca0b0d4254493"

 [metadata.files]
 adal = [
--- a/pyproject.toml
+++ b/pyproject.toml
@ -12,7 +12,7 @@ documentation = "https://docs.comma.ai"
 [tool.poetry.dependencies]
 python = "~3.8"
 atomicwrites = "^1.4.0"
-casadi = { version = "==3.6.3", platform = "linux" }
+casadi = "==3.6.3"
 cffi = "^1.15.1"
 crcmod = "^1.7"
 cryptography = "^37.0.4"
--- a/release/files_common
+++ b/release/files_common
@ -439,6 +439,7 @@ third_party/libyuv/larch64/**
 third_party/snpe/include/**
 third_party/snpe/dsp**

+third_party/acados/.gitignore
 third_party/acados/x86_64/**
 third_party/acados/larch64/**
 third_party/acados/include/**
--- a/selfdrive/locationd/test/_test_locationd_lib.py
+++ b/selfdrive/locationd/test/_test_locationd_lib.py
@ -10,10 +10,12 @@ from cffi import FFI
 import cereal.messaging as messaging
 from cereal import log

+from common.ffi_wrapper import suffix
+
 SENSOR_DECIMATION = 1
 VISION_DECIMATION = 1

-LIBLOCATIOND_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '../liblocationd.so'))
+LIBLOCATIOND_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '../liblocationd' + suffix()))


 class TestLocationdLib(unittest.TestCase):
--- a/third_party/acados/.gitignore
+++ b/third_party/acados/.gitignore
@ -1,4 +1,5 @@
 acados_repo/
+lib
 !x86_64/
 !larch64/
 !aarch64/
--- a/third_party/acados/Darwin/lib/libacados.dylib
+++ b/third_party/acados/Darwin/lib/libacados.dylib
--- a/third_party/acados/Darwin/lib/libblasfeo.dylib
+++ b/third_party/acados/Darwin/lib/libblasfeo.dylib
--- a/third_party/acados/Darwin/lib/libhpipm.dylib
+++ b/third_party/acados/Darwin/lib/libhpipm.dylib
--- a/third_party/acados/Darwin/lib/libqpOASES_e.3.1.dylib
+++ b/third_party/acados/Darwin/lib/libqpOASES_e.3.1.dylib
--- a/third_party/acados/Darwin/t_renderer
+++ b/third_party/acados/Darwin/t_renderer
--- a/third_party/acados/build.sh
+++ b/third_party/acados/build.sh
@ -13,13 +13,8 @@ fi
 ACADOS_FLAGS="-DACADOS_WITH_QPOASES=ON -UBLASFEO_TARGET -DBLASFEO_TARGET=$BLAS_TARGET"

 if [[ "$OSTYPE" == "darwin"* ]]; then
-  if [[  $(uname -m) == "x86_64" ]]; then
-    ACADOS_FLAGS="$ACADOS_FLAGS -DCMAKE_OSX_ARCHITECTURES=x86_64"
-    ARCHNAME="Darwin_x86_64"
-  else
-    ACADOS_FLAGS="$ACADOS_FLAGS -DCMAKE_OSX_ARCHITECTURES=arm64;x86_64"
-    ARCHNAME="Darwin"
-  fi
+  ACADOS_FLAGS="$ACADOS_FLAGS -DCMAKE_OSX_ARCHITECTURES=arm64;x86_64 -DCMAKE_MACOSX_RPATH=1"
+  ARCHNAME="Darwin"
 fi

 if [ ! -d acados_repo/ ]; then
@ -51,5 +46,11 @@ cp -r $DIR/acados_repo/interfaces/acados_template/acados_template $DIR/

 # build tera
 cd $DIR/acados_repo/interfaces/acados_template/tera_renderer/
-cargo build --verbose --release
+if [[ "$OSTYPE" == "darwin"* ]]; then
+  cargo build --verbose --release --target aarch64-apple-darwin
+  cargo build --verbose --release --target x86_64-apple-darwin
+  lipo -create -output target/release/t_renderer target/x86_64-apple-darwin/release/t_renderer target/aarch64-apple-darwin/release/t_renderer
+else 
+  cargo build --verbose --release
+fi
 cp target/release/t_renderer $INSTALL_DIR/
--- a/tools/cabana/util.cc
+++ b/tools/cabana/util.cc
@ -1,5 +1,7 @@
 #include "tools/cabana/util.h"

+#include <array>
+
 #include <QColor>
 #include <QFontDatabase>
 #include <QLocale>
--- a/tools/mac_setup.sh
+++ b/tools/mac_setup.sh
@ -69,6 +69,7 @@ export CPPFLAGS="$CPPFLAGS -I${BREW_PREFIX}/opt/bzip2/include"
 # pycurl curl/openssl backend dependencies
 export LDFLAGS="$LDFLAGS -L${BREW_PREFIX}/opt/openssl@3/lib"
 export CPPFLAGS="$CPPFLAGS -I${BREW_PREFIX}/opt/openssl@3/include"
+export PYCURL_CURL_CONFIG=/usr/bin/curl-config
 export PYCURL_SSL_LIBRARY=openssl

 # openpilot environment
@ -83,29 +84,6 @@ $ROOT/update_requirements.sh
 eval "$(pyenv init --path)"
 echo "[ ] installed python dependencies t=$SECONDS"

-# install casadi
-VENV=`poetry env info --path`
-PYTHON_VER=3.8
-PYTHON_VERSION=$(cat $ROOT/.python-version)
-if [ ! -f "$VENV/include/casadi/casadi.hpp" ]; then
-  echo "-- casadi manual install"
-  cd /tmp/ && curl -L https://github.com/casadi/casadi/archive/refs/tags/ge6.tar.gz --output casadi.tar.gz
-  tar -xzf casadi.tar.gz
-  cd casadi-ge6/ && mkdir -p build && cd build
-  cmake .. \
-    -DWITH_PYTHON=ON \
-    -DWITH_EXAMPLES=OFF \
-    -DCMAKE_INSTALL_PREFIX:PATH=$VENV \
-    -DPYTHON_PREFIX:PATH=$VENV/lib/python$PYTHON_VER/site-packages \
-    -DPYTHON_LIBRARY:FILEPATH=$HOME/.pyenv/versions/$PYTHON_VERSION/lib/libpython$PYTHON_VER.dylib \
-    -DPYTHON_EXECUTABLE:FILEPATH=$HOME/.pyenv/versions/$PYTHON_VERSION/bin/python \
-    -DPYTHON_INCLUDE_DIR:PATH=$HOME/.pyenv/versions/$PYTHON_VERSION/include/python$PYTHON_VER \
-    -DCMAKE_CXX_FLAGS="-ferror-limit=0" -DCMAKE_C_FLAGS="-ferror-limit=0"
-  CFLAGS="-ferror-limit=0" make -j$(nproc) && make install
-else
-  echo "----   casadi found in venv. skipping build   ----"
-fi
-
 echo
 echo "----   OPENPILOT SETUP DONE   ----"
 echo "Open a new shell or configure your active shell env by running:"