mac: fix setup and re-enable CI (#23365)

* rework mac_setup.sh
* ignore casadi and onnx runtime in pipfile for Darwin
* remove rust stuff
* build and install casadi

* add wget

* explicit python3. dont do onnx just yet

* resource compiler

* replace acados Darwin lib w universal2 libs

* relock Pipfile using linux machine

* Update update_requirements.sh

brackets

* Update update_requirements.sh

oops

* ci: re-enable mac build

* attempt to fix ci build:
* pip stuff?
* move rcc bin path to env script

* oops

* only mac

* k im lazy. does ci like this??

* huh??

* * use curl
* avoid casadi rebuilds
* add comment to remove protobuf
* host detect typo

* python version

* how did the builds not use swig?
* better reinstall check

* sometimes mac clang complain abt error limit

* ci: build OP in venv

* ci: bump to Big Sur
* retry? idk why it failed @ acados link

* * use macos-latest
* move rcc path to scons
* add extra paths for odd homebrew installs
* acados source change

* update macOS README

* uh, maybe?

* k nvm

* ci: ok this is strange. might be scons bug?

* fix conflicts: bump

* just add cppcheck for pre-commit stuff

* agane

* cleanup

* try that

* fix path

* no pyenv update on mac

* source

* fix rpath prefix?

* no examples

* fix exit

* let's get this cached

* add virtualenvs to cache

* why did we cache that

* let's see what's big

* more

* always ruyn

* cache scons cache

* better cache key

* fix for partial hit

* why so long

Co-authored-by: Andrew Tec <andrest@trabus.com>
Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>
Co-authored-by: Bruce Wayne <batman@openpilot-mac-mini.local>
This commit is contained in:
Andrew 2022-01-07 15:15:18 -08:00 committed by GitHub
parent 207ce410b4
commit 6664421710
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 158 additions and 92 deletions

View File

@ -69,52 +69,62 @@ jobs:
rm -rf /tmp/scons_cache/* && \
scons -j$(nproc) --cache-populate"
#build_mac:
# name: build macos
# runs-on: macos-10.15
# timeout-minutes: 60
# steps:
# - uses: actions/checkout@v2
# with:
# submodules: true
# - name: Determine pre-existing Homebrew packages
# if: steps.dependency-cache.outputs.cache-hit != 'true'
# run: |
# echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
# ls -1 /usr/local/Cellar >> $GITHUB_ENV
# echo 'EOF' >> $GITHUB_ENV
# - name: Cache dependencies
# id: dependency-cache
# uses: actions/cache@v2
# with:
# path: |
# ~/.pyenv
# ~/Library/Caches/pip
# ~/Library/Caches/pipenv
# /usr/local/Cellar
# ~/github_brew_cache_entries.txt
# key: macos-cache-${{ hashFiles('tools/mac_setup.sh') }}
# - name: Brew link restored dependencies
# if: steps.dependency-cache.outputs.cache-hit == 'true'
# run: |
# while read pkg; do
# brew link --force "$pkg" # `--force` for keg-only packages
# done < ~/github_brew_cache_entries.txt
# - name: Install dependencies
# run: ./tools/mac_setup.sh
# - name: Build openpilot
# run: eval "$(pyenv init -)" && scons -j$(nproc)
# - name: Remove pre-existing Homebrew packages for caching
# if: steps.dependency-cache.outputs.cache-hit != 'true'
# run: |
# cd /usr/local/Cellar
# new_cellar=$(ls -1)
# comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do
# if [[ $pkg != "zstd" ]]; then # caching step needs zstd
# rm -rf "$pkg"
# fi
# done
# comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt
build_mac:
name: build macos
runs-on: macos-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Determine pre-existing Homebrew packages
if: steps.dependency-cache.outputs.cache-hit != 'true'
run: |
echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
ls -1 /usr/local/Cellar >> $GITHUB_ENV
echo 'EOF' >> $GITHUB_ENV
- name: Cache dependencies
id: dependency-cache
uses: actions/cache@v2
with:
path: |
~/.pyenv
~/.local/share/virtualenvs/
/usr/local/Cellar
~/github_brew_cache_entries.txt
/tmp/scons_cache
key: macos-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'Pipfile*') }}
restore-keys: macos-
- name: Brew link restored dependencies
run: |
if [ -f ~/github_brew_cache_entries.txt ]; then
while read pkg; do
brew link --force "$pkg" # `--force` for keg-only packages
done < ~/github_brew_cache_entries.txt
else
echo "Cache entries not found"
fi
- name: Install dependencies
run: ./tools/mac_setup.sh
- name: Build openpilot
run: |
source tools/openpilot_env.sh
pipenv run selfdrive/manager/build.py
# cleanup scons cache
rm -rf /tmp/scons_cache/
pipenv run scons -j$(nproc) --cache-populate
- name: Remove pre-existing Homebrew packages for caching
if: steps.dependency-cache.outputs.cache-hit != 'true'
run: |
cd /usr/local/Cellar
new_cellar=$(ls -1)
comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do
if [[ $pkg != "zstd" ]]; then # caching step needs zstd
rm -rf "$pkg"
fi
done
comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt
build_webcam:
name: build webcam

View File

@ -38,7 +38,7 @@ tenacity = "*"
[packages]
atomicwrites = "*"
casadi = "*"
casadi = {version = "*", markers="platform_system != 'Darwin'"}
cffi = "*"
crcmod = "*"
cryptography = "*"
@ -54,7 +54,7 @@ libusb1 = "*"
nose = "*"
numpy = "*"
onnx = "*"
onnxruntime-gpu = "*"
onnxruntime-gpu = {version = "*", markers="platform_system != 'Darwin'"}
pillow = "*"
psutil = "*"
pycapnp = "==1.1.0"

60
Pipfile.lock generated
View File

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "658c6ab1fd200a6c84b1dec7a3b15b3d01ce911a76b1201bd4495b3031eac119"
"sha256": "40d9fa44e5b593786d40aa8a0276a1fbdfac46fbb78734687e6ef5b870813ed8"
},
"pipfile-spec": 6,
"requires": {
@ -81,6 +81,7 @@
"sha256:fbf39dcd63f1d3b63c300fce59b7ea678bd5ea1d014e1e090a5226600a4132cb"
],
"index": "pypi",
"markers": "platform_system != 'Darwin'",
"version": "==3.5.5"
},
"certifi": {
@ -148,11 +149,11 @@
},
"charset-normalizer": {
"hashes": [
"sha256:1eecaa09422db5be9e29d7fc65664e6c33bd06f9ced7838578ba40d58bdf3721",
"sha256:b0b883e8e874edfdece9c28f314e3dd5badf067342e42fb162203335ae61aa2c"
"sha256:876d180e9d7432c5d1dfd4c5d26b72f099d503e8fcc0feb7532c9289be60fcbd",
"sha256:cb957888737fc0bbcd78e3df769addb41fd1ff8cf950dc9e7ad7793f1bf44455"
],
"markers": "python_version >= '3'",
"version": "==2.0.9"
"version": "==2.0.10"
},
"click": {
"hashes": [
@ -309,7 +310,7 @@
"sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7",
"sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951"
],
"markers": "python_full_version >= '3.6.1' and python_version < '4.0'",
"markers": "python_version < '4.0' and python_full_version >= '3.6.1'",
"version": "==5.10.1"
},
"itsdangerous": {
@ -558,6 +559,7 @@
"sha256:f6e056cd38265e1a158b72da234546291bb3cad663d81dcd66cbcbbce028e3a8"
],
"index": "pypi",
"markers": "platform_system != 'Darwin'",
"version": "==1.10.0"
},
"pillow": {
@ -893,11 +895,11 @@
},
"requests": {
"hashes": [
"sha256:8e5643905bf20a308e25e4c1dd379117c09000bf8a82ebccc462cfb1b34a16b5",
"sha256:f71a09d7feba4a6b64ffd8e9d9bc60f9bf7d7e19fd0e04362acb1cfc2e3d98df"
"sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61",
"sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"
],
"index": "pypi",
"version": "==2.27.0"
"version": "==2.27.1"
},
"scons": {
"hashes": [
@ -942,6 +944,14 @@
"index": "pypi",
"version": "==1.2.2"
},
"setuptools": {
"hashes": [
"sha256:5c89b1a14a67ac5f0956f1cb0aeb7d1d3f4c8ba4e4e1ab7bf1af4933f9a2f0fe",
"sha256:675fcebecb43c32eb930481abf907619137547f4336206e4d673180242e1a278"
],
"markers": "python_version >= '3.7'",
"version": "==60.2.0"
},
"six": {
"hashes": [
"sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
@ -979,7 +989,7 @@
"sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
"sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
],
"markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'",
"markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==0.10.2"
},
"tqdm": {
@ -995,7 +1005,7 @@
"sha256:4ca091dea149f945ec56afb48dae714f21e8692ef22a395223bcd328961b6a0e",
"sha256:7f001e5ac290a0c0401508864c7ec868be4e701886d5b573a9528ed3973d9d3b"
],
"markers": "python_version < '3.10' and python_version < '3.10'",
"markers": "python_version < '3.10'",
"version": "==4.0.1"
},
"urllib3": {
@ -1208,11 +1218,11 @@
},
"charset-normalizer": {
"hashes": [
"sha256:1eecaa09422db5be9e29d7fc65664e6c33bd06f9ced7838578ba40d58bdf3721",
"sha256:b0b883e8e874edfdece9c28f314e3dd5badf067342e42fb162203335ae61aa2c"
"sha256:876d180e9d7432c5d1dfd4c5d26b72f099d503e8fcc0feb7532c9289be60fcbd",
"sha256:cb957888737fc0bbcd78e3df769addb41fd1ff8cf950dc9e7ad7793f1bf44455"
],
"markers": "python_version >= '3'",
"version": "==2.0.9"
"version": "==2.0.10"
},
"control": {
"hashes": [
@ -1387,11 +1397,11 @@
},
"hypothesis": {
"hashes": [
"sha256:3cae408a92917fe288b0932eb3bb758f1585bf8b6e1d4ca4a666a2835c707db3",
"sha256:5002ce1f27fd94b53b0046ac7d20dab17fd74ebc6c90d401979588eadc46f84e"
"sha256:317f8d2f670fa69e258ab43e21c2befd413c559e386581f7e9641a80460b1063",
"sha256:803792d416ff71307d775fe760e2b2f07ca302a2c941b576629668092b9f3e3d"
],
"index": "pypi",
"version": "==6.34.1"
"version": "==6.34.2"
},
"identify": {
"hashes": [
@ -2032,11 +2042,11 @@
},
"requests": {
"hashes": [
"sha256:8e5643905bf20a308e25e4c1dd379117c09000bf8a82ebccc462cfb1b34a16b5",
"sha256:f71a09d7feba4a6b64ffd8e9d9bc60f9bf7d7e19fd0e04362acb1cfc2e3d98df"
"sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61",
"sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"
],
"index": "pypi",
"version": "==2.27.0"
"version": "==2.27.1"
},
"reverse-geocoder": {
"hashes": [
@ -2080,6 +2090,14 @@
"index": "pypi",
"version": "==1.7.3"
},
"setuptools": {
"hashes": [
"sha256:5c89b1a14a67ac5f0956f1cb0aeb7d1d3f4c8ba4e4e1ab7bf1af4933f9a2f0fe",
"sha256:675fcebecb43c32eb930481abf907619137547f4336206e4d673180242e1a278"
],
"markers": "python_version >= '3.7'",
"version": "==60.2.0"
},
"six": {
"hashes": [
"sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
@ -2195,7 +2213,7 @@
"sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
"sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
],
"markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'",
"markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==0.10.2"
},
"tomli": {
@ -2211,7 +2229,7 @@
"sha256:4ca091dea149f945ec56afb48dae714f21e8692ef22a395223bcd328961b6a0e",
"sha256:7f001e5ac290a0c0401508864c7ec868be4e701886d5b573a9528ed3973d9d3b"
],
"markers": "python_version < '3.10' and python_version < '3.10'",
"markers": "python_version < '3.10'",
"version": "==4.0.1"
},
"urllib3": {

View File

@ -66,7 +66,7 @@ lenv = {
"LD_LIBRARY_PATH": [Dir(f"#third_party/acados/{arch}/lib").abspath],
"PYTHONPATH": Dir("#").abspath + ":" + Dir("#pyextra/").abspath,
"ACADOS_SOURCE_DIR": Dir("#third_party/acados/acados").abspath,
"ACADOS_SOURCE_DIR": Dir("#third_party/acados/include/acados").abspath,
"ACADOS_PYTHON_INTERFACE_PATH": Dir("#pyextra/acados_template").abspath,
"TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer",
}
@ -125,8 +125,10 @@ else:
f"#third_party/libyuv/{yuv_dir}/lib",
"/usr/local/lib",
"/opt/homebrew/lib",
"/usr/local/Homebrew/Library",
"/usr/local/opt/openssl/lib",
"/opt/homebrew/opt/openssl/lib",
"/usr/local/Cellar",
f"#third_party/acados/{arch}/lib",
"/System/Library/Frameworks/OpenGL.framework/Libraries",
]
@ -134,6 +136,7 @@ else:
cxxflags += ["-DGL_SILENCE_DEPRECATION"]
cpppath += [
"/opt/homebrew/include",
"/usr/local/include",
"/usr/local/opt/openssl/include",
"/opt/homebrew/opt/openssl/include"
]
@ -235,6 +238,9 @@ env = Environment(
tools=["default", "cython", "compilation_db"],
)
if arch == "Darwin":
env['RPATHPREFIX'] = "-rpath "
if GetOption('compile_db'):
env.CompilationDatabase('compile_commands.json')
@ -299,6 +305,7 @@ if arch == "Darwin":
qt_dirs += [f"{qt_env['QTDIR']}/include/Qt{m}" for m in qt_modules]
qt_env["LINKFLAGS"] += ["-F" + os.path.join(qt_env['QTDIR'], "lib")]
qt_env["FRAMEWORKS"] += [f"Qt{m}" for m in qt_modules] + ["OpenGL"]
qt_env.AppendENVPath('PATH', os.path.join(qt_env['QTDIR'], "bin"))
elif arch == "aarch64":
qt_env['QTDIR'] = "/system/comma/usr"
qt_dirs = [

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@ -1,4 +1,6 @@
#!/bin/bash -e
#!/bin/bash
set -e
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
ROOT="$(cd $DIR/../ && pwd)"
@ -7,15 +9,16 @@ ROOT="$(cd $DIR/../ && pwd)"
if [[ $(command -v brew) == "" ]]; then
echo "Installing Hombrew"
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
echo "[ ] installed brew t=$SECONDS"
fi
# TODO: remove protobuf,protobuf-c,swig when casadi can be pip installed
brew bundle --file=- <<-EOS
brew "cmake"
brew "cppcheck"
brew "git-lfs"
brew "zlib"
brew "bzip2"
brew "rust"
brew "rustup-init"
brew "capnp"
brew "coreutils"
brew "eigen"
@ -29,21 +32,20 @@ brew "openssl"
brew "pyenv"
brew "qt@5"
brew "zeromq"
brew "protobuf"
brew "protobuf-c"
brew "swig"
cask "gcc-arm-embedded"
EOS
echo "[ ] finished brew install t=$SECONDS"
if [[ $SHELL == "/bin/zsh" ]]; then
RC_FILE="$HOME/.zshrc"
elif [[ $SHELL == "/bin/bash" ]]; then
RC_FILE="$HOME/.bash_profile"
fi
# TODO: get rid of this somehow
# Build requirements for macOS
# https://github.com/pyenv/pyenv/issues/1740
# https://github.com/pyca/cryptography/blob/main/docs/installation.rst
rustup-init -y
export LDFLAGS="$LDFLAGS -L/usr/local/opt/zlib/lib"
export LDFLAGS="$LDFLAGS -L/usr/local/opt/bzip2/lib"
export LDFLAGS="$LDFLAGS -L/usr/local/opt/openssl@1.1/lib"
@ -55,17 +57,40 @@ export PATH="$PATH:/usr/local/bin"
# openpilot environment
if [ -z "$OPENPILOT_ENV" ] && [ -n "$RC_FILE" ] && [ -z "$CI" ]; then
echo "export PATH=\"\$PATH:$HOME/.cargo/bin\"" >> $RC_FILE
echo "source $ROOT/tools/openpilot_env.sh" >> $RC_FILE
export PATH="$PATH:\"\$HOME/.cargo/bin\""
source "$ROOT/tools/openpilot_env.sh"
echo "Added openpilot_env to RC file: $RC_FILE"
fi
# install python dependencies
$ROOT/update_requirements.sh
eval "$(pyenv init --path)"
echo "[ ] installed python dependencies t=$SECONDS"
# install casadi
VENV=`pipenv --venv`
PYTHON_VER=3.8
PYTHON_VERSION=$(cat $ROOT/.python-version)
if [ ! -f "$VENV/include/casadi/casadi.hpp" ]; then
echo "-- casadi manual install"
cd /tmp/ && curl -L https://github.com/casadi/casadi/archive/refs/tags/ge6.tar.gz --output casadi.tar.gz
tar -xzf casadi.tar.gz
cd casadi-ge6/ && mkdir -p build && cd build
cmake .. \
-DWITH_PYTHON=ON \
-DWITH_EXAMPLES=OFF \
-DCMAKE_INSTALL_PREFIX:PATH=$VENV \
-DPYTHON_PREFIX:PATH=$VENV/lib/python$PYTHON_VER/site-packages \
-DPYTHON_LIBRARY:FILEPATH=$HOME/.pyenv/versions/$PYTHON_VERSION/lib/libpython$PYTHON_VER.dylib \
-DPYTHON_EXECUTABLE:FILEPATH=$HOME/.pyenv/versions/$PYTHON_VERSION/bin/python \
-DPYTHON_INCLUDE_DIR:PATH=$HOME/.pyenv/versions/$PYTHON_VERSION/include/python$PYTHON_VER \
-DCMAKE_CXX_FLAGS="-ferror-limit=0" -DCMAKE_C_FLAGS="-ferror-limit=0"
CFLAGS="-ferror-limit=0" make -j$(nproc) && make install
else
echo "---- casadi found in venv. skipping build ----"
fi
echo
echo "---- FINISH OPENPILOT SETUP ----"
echo "Configure your active shell env by running:"
echo "---- OPENPILOT SETUP DONE ----"
echo "Open a new shell or configure your active shell env by running:"
echo "source $RC_FILE"

View File

@ -9,13 +9,13 @@ if [ -z "$OPENPILOT_ENV" ]; then
export PYENV_ROOT="$HOME/.pyenv"
if [[ "$(uname)" == 'Linux' ]]; then
eval "$(pyenv init --path)"
eval "$(pyenv virtualenv-init -)"
elif [[ "$(uname)" == 'Darwin' ]]; then
# msgq doesn't work on mac
export ZMQ=1
export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
fi
eval "$(pyenv init --path)"
eval "$(pyenv init -)"
export OPENPILOT_ENV=1

View File

@ -134,6 +134,6 @@ if [ -z "$OPENPILOT_ENV" ]; then
fi
echo
echo "---- FINISH OPENPILOT SETUP ----"
echo "Configure your active shell env by running:"
echo "---- OPENPILOT SETUP DONE ----"
echo "Open a new shell or configure your active shell env by running:"
echo "source ~/.bashrc"

View File

@ -1,4 +1,6 @@
#!/bin/bash -e
#!/bin/bash
set -e
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
cd $DIR
@ -13,15 +15,18 @@ export MAKEFLAGS="-j$(nproc)"
PYENV_PYTHON_VERSION=$(cat .python-version)
if ! pyenv prefix ${PYENV_PYTHON_VERSION} &> /dev/null; then
echo "pyenv update ..."
pyenv update
# no pyenv update on mac
if [ "$(uname)" == "Linux" ]; then
echo "pyenv update ..."
pyenv update
fi
echo "python ${PYENV_PYTHON_VERSION} install ..."
CONFIGURE_OPTS="--enable-shared" pyenv install -f ${PYENV_PYTHON_VERSION}
fi
eval "$(pyenv init --path)"
echo "pip install ..."
echo "update pip"
pip install pip==21.3.1
echo "pipenv install ..."
pip install pipenv==2021.11.23
if [ -d "./xx" ]; then
@ -36,7 +41,7 @@ else
RUN=""
fi
echo "pip packages install ..."
echo "pip packages install..."
pipenv install --dev --deploy --clear
pyenv rehash
@ -45,4 +50,5 @@ if [ -f "$DIR/.pre-commit-config.yaml" ]; then
$RUN pre-commit install
[ -d "./xx" ] && (cd xx && $RUN pre-commit install)
[ -d "./notebooks" ] && (cd notebooks && $RUN pre-commit install)
echo "pre-commit hooks installed"
fi