mirror of https://github.com/commaai/cereal.git
Run scons in CI (#14)
* try to run scons in azure pipelines * sudo * install capnp * does this run * also clean test runner * remove makefiles * this should build
This commit is contained in:
Willem Melching
GitHub
parent
9414615b99
commit
52c6db8719
|
|
@ -0,0 +1 @@
|
|||
.sconsign.dblite
|
||||
|
|
@ -9,4 +9,5 @@ libcereal*.a
|
|||
libmessaging.*
|
||||
libmessaging_shared.*
|
||||
services.h
|
||||
.sconsign.dblite
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
from ubuntu:16.04
|
||||
|
||||
RUN apt-get update && apt-get install -y libzmq3-dev clang wget git autoconf libtool curl make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python-openssl
|
||||
|
||||
RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
|
||||
ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}"
|
||||
RUN pyenv install 3.7.3
|
||||
RUN pyenv global 3.7.3
|
||||
RUN pyenv rehash
|
||||
RUN pip3 install pyyaml==5.1.2 Cython==0.29.14 scons==3.1.1 pycapnp==0.6.4
|
||||
|
||||
WORKDIR /project/cereal
|
||||
COPY install_capnp.sh .
|
||||
RUN ./install_capnp.sh
|
||||
|
||||
ENV PYTHONPATH=/project
|
||||
|
||||
COPY . .
|
||||
RUN scons -c && scons -j$(nproc)
|
||||
62
Makefile
62
Makefile
|
|
@ -1,62 +0,0 @@
|
|||
PWD := $(shell pwd)
|
||||
|
||||
SRCS := log.capnp car.capnp
|
||||
|
||||
GENS := gen/cpp/car.capnp.c++ gen/cpp/log.capnp.c++
|
||||
JS := gen/js/car.capnp.js gen/js/log.capnp.js
|
||||
|
||||
UNAME_M ?= $(shell uname -m)
|
||||
|
||||
GENS += gen/c/car.capnp.c gen/c/log.capnp.c gen/c/include/c++.capnp.h gen/c/include/java.capnp.h
|
||||
|
||||
ifeq ($(UNAME_M),x86_64)
|
||||
|
||||
ifneq (, $(shell which capnpc-java))
|
||||
GENS += gen/java/Car.java gen/java/Log.java
|
||||
else
|
||||
$(warning capnpc-java not found, skipping java build)
|
||||
endif
|
||||
|
||||
endif
|
||||
|
||||
|
||||
ifeq ($(UNAME_M),aarch64)
|
||||
CAPNPC=PATH=$(PWD)/../phonelibs/capnp-cpp/aarch64/bin/:$$PATH capnpc
|
||||
else
|
||||
CAPNPC=capnpc
|
||||
endif
|
||||
|
||||
.PHONY: all
|
||||
all: $(GENS)
|
||||
js: $(JS)
|
||||
|
||||
.PHONY: clean
|
||||
clean:
|
||||
rm -rf gen
|
||||
rm -rf node_modules
|
||||
rm -rf package-lock.json
|
||||
|
||||
gen/c/%.capnp.c: %.capnp
|
||||
@echo "[ CAPNPC C ] $@"
|
||||
mkdir -p gen/c/
|
||||
$(CAPNPC) '$<' -o c:gen/c/
|
||||
|
||||
gen/js/%.capnp.js: %.capnp
|
||||
@echo "[ CAPNPC JavaScript ] $@"
|
||||
mkdir -p gen/js/
|
||||
sh ./generate_javascript.sh
|
||||
|
||||
gen/cpp/%.capnp.c++: %.capnp
|
||||
@echo "[ CAPNPC C++ ] $@"
|
||||
mkdir -p gen/cpp/
|
||||
$(CAPNPC) '$<' -o c++:gen/cpp/
|
||||
|
||||
gen/java/Car.java gen/java/Log.java: $(SRCS)
|
||||
@echo "[ CAPNPC java ] $@"
|
||||
mkdir -p gen/java/
|
||||
$(CAPNPC) $^ -o java:gen/java
|
||||
|
||||
# c-capnproto needs some empty headers
|
||||
gen/c/include/c++.capnp.h gen/c/include/java.capnp.h:
|
||||
mkdir -p gen/c/include
|
||||
touch '$@'
|
||||
21
SConscript
21
SConscript
|
|
@ -1,15 +1,18 @@
|
|||
Import('env', 'arch', 'zmq')
|
||||
|
||||
gen_dir = Dir('gen')
|
||||
messaging_dir = Dir('messaging')
|
||||
|
||||
# TODO: remove src-prefix and cereal from command string. can we set working directory?
|
||||
env.Command(["gen/c/include/c++.capnp.h", "gen/c/include/java.capnp.h"], [], "mkdir -p cereal/gen/c/include && touch $TARGETS")
|
||||
env.Command(["gen/c/include/c++.capnp.h", "gen/c/include/java.capnp.h"], [], "mkdir -p " + gen_dir.path + "/c/include && touch $TARGETS")
|
||||
env.Command(
|
||||
['gen/c/car.capnp.c', 'gen/c/log.capnp.c', 'gen/c/car.capnp.h', 'gen/c/log.capnp.h'],
|
||||
['car.capnp', 'log.capnp'],
|
||||
'capnpc $SOURCES --src-prefix=cereal -o c:cereal/gen/c/')
|
||||
'capnpc $SOURCES --src-prefix=cereal -o c:' + gen_dir.path + '/c/')
|
||||
env.Command(
|
||||
['gen/cpp/car.capnp.c++', 'gen/cpp/log.capnp.c++', 'gen/cpp/car.capnp.h', 'gen/cpp/log.capnp.h'],
|
||||
['car.capnp', 'log.capnp'],
|
||||
'capnpc $SOURCES --src-prefix=cereal -o c++:cereal/gen/cpp/')
|
||||
'capnpc $SOURCES --src-prefix=cereal -o c++:' + gen_dir.path + '/cpp/')
|
||||
|
||||
env.Library('cereal', [
|
||||
'gen/c/car.capnp.c',
|
||||
|
|
@ -18,10 +21,12 @@ env.Library('cereal', [
|
|||
'gen/cpp/log.capnp.c++',
|
||||
])
|
||||
|
||||
|
||||
cereal_dir = Dir('.')
|
||||
env.Command(
|
||||
['services.h'],
|
||||
['service_list.yaml', 'services.py'],
|
||||
'python3 cereal/services.py > $TARGET')
|
||||
'python3 ' + cereal_dir.path + '/services.py > $TARGET')
|
||||
|
||||
messaging_deps = [
|
||||
'messaging/messaging.cc',
|
||||
|
|
@ -39,12 +44,16 @@ if arch == "aarch64":
|
|||
messaging_shared_lib = env.SharedLibrary('messaging_shared', messaging_deps, LIBS=shared_lib_shared_lib)
|
||||
env.Command(['messaging/messaging.so'], [messaging_shared_lib], "chmod 777 $SOURCES && ln -sf `realpath $SOURCES` $TARGET")
|
||||
|
||||
env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=['messaging', 'zmq'])
|
||||
env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=[messaging_lib, 'zmq'])
|
||||
|
||||
# different target?
|
||||
#env.Program('messaging/demo', ['messaging/demo.cc'], LIBS=['messaging', 'zmq'])
|
||||
|
||||
|
||||
env.Command(['messaging/messaging_pyx.so'],
|
||||
[messaging_lib, 'messaging/messaging_pyx_setup.py', 'messaging/messaging_pyx.pyx', 'messaging/messaging.pxd'],
|
||||
"cd cereal/messaging && python3 messaging_pyx_setup.py build_ext --inplace")
|
||||
"cd " + messaging_dir.path + " && python3 messaging_pyx_setup.py build_ext --inplace")
|
||||
|
||||
|
||||
if GetOption('test'):
|
||||
env.Program('messaging/test_runner', ['messaging/test_runner.cc', 'messaging/msgq_tests.cc'], LIBS=[messaging_lib])
|
||||
|
|
|
|||
|
|
@ -0,0 +1,49 @@
|
|||
import os
|
||||
import subprocess
|
||||
|
||||
zmq = 'zmq'
|
||||
arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
|
||||
|
||||
cereal_dir = Dir('.')
|
||||
|
||||
cpppath = [
|
||||
cereal_dir,
|
||||
'/usr/lib/include',
|
||||
]
|
||||
|
||||
AddOption('--test',
|
||||
action='store_true',
|
||||
help='build test files')
|
||||
|
||||
AddOption('--asan',
|
||||
action='store_true',
|
||||
help='turn on ASAN')
|
||||
|
||||
ccflags_asan = ["-fsanitize=address", "-fno-omit-frame-pointer"] if GetOption('asan') else []
|
||||
ldflags_asan = ["-fsanitize=address"] if GetOption('asan') else []
|
||||
|
||||
env = Environment(
|
||||
ENV=os.environ,
|
||||
CC='clang',
|
||||
CXX='clang++',
|
||||
CCFLAGS=[
|
||||
"-g",
|
||||
"-fPIC",
|
||||
"-O2",
|
||||
"-Werror=implicit-function-declaration",
|
||||
"-Werror=incompatible-pointer-types",
|
||||
"-Werror=int-conversion",
|
||||
"-Werror=return-type",
|
||||
"-Werror=format-extra-args",
|
||||
] + ccflags_asan,
|
||||
LDFLAGS=ldflags_asan,
|
||||
LINKFLAGS=ldflags_asan,
|
||||
|
||||
CFLAGS="-std=gnu11",
|
||||
CXXFLAGS="-std=c++14",
|
||||
CPPPATH=cpppath,
|
||||
)
|
||||
|
||||
|
||||
Export('env', 'zmq', 'arch')
|
||||
SConscript(['SConscript'])
|
||||
|
|
@ -3,10 +3,7 @@ pool:
|
|||
|
||||
steps:
|
||||
- script: |
|
||||
cd messaging
|
||||
ASAN=1 make test_runner
|
||||
./test_runner -r junit -o tests.xml
|
||||
docker build -t cereal .
|
||||
docker run cereal bash -c "scons --test --asan -j$(nproc) && messaging/test_runner"
|
||||
|
||||
displayName: 'Run Tests'
|
||||
- task: PublishTestResults@2
|
||||
inputs:
|
||||
testResultsFiles: 'messaging/tests.xml'
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@ tar xvf capnproto-c++-${VERSION}.tar.gz
|
|||
cd capnproto-c++-${VERSION}
|
||||
CXXFLAGS="-fPIC" ./configure
|
||||
|
||||
make -j4
|
||||
make -j$(nproc)
|
||||
make install
|
||||
|
||||
# manually build binaries statically
|
||||
g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -DCAPNP_INCLUDE_DIR=\"/usr/local/include\" -pthread -O2 -DNDEBUG -pthread -pthread -o .libs/capnp src/capnp/compiler/module-loader.o src/capnp/compiler/capnp.o ./.libs/libcapnpc.a ./.libs/libcapnp.a ./.libs/libkj.a -lpthread -pthread
|
||||
|
|
@ -18,7 +19,6 @@ g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -D
|
|||
g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -DCAPNP_INCLUDE_DIR=\"/usr/local/include\" -pthread -O2 -DNDEBUG -pthread -pthread -o .libs/capnpc-capnp src/capnp/compiler/capnpc-capnp.o ./.libs/libcapnp.a ./.libs/libkj.a -lpthread -pthread
|
||||
|
||||
cp .libs/capnp /usr/local/bin/
|
||||
ln -s /usr/local/bin/capnp /usr/local/bin/capnpc
|
||||
cp .libs/capnpc-c++ /usr/local/bin/
|
||||
cp .libs/capnpc-capnp /usr/local/bin/
|
||||
cp .libs/*.a /usr/local/lib
|
||||
|
|
@ -30,7 +30,8 @@ cd c-capnproto
|
|||
git submodule update --init --recursive
|
||||
autoreconf -f -i -s
|
||||
CXXFLAGS="-fPIC" ./configure
|
||||
make -j4
|
||||
make -j$(nproc)
|
||||
make install
|
||||
|
||||
# manually build binaries statically
|
||||
gcc -fPIC -o .libs/capnpc-c compiler/capnpc-c.o compiler/schema.capnp.o compiler/str.o ./.libs/libcapnp_c.a
|
||||
|
|
|
|||
|
|
@ -7,3 +7,4 @@ test_runner
|
|||
*.a
|
||||
*.so
|
||||
messaging_pyx.cpp
|
||||
build/
|
||||
|
|
|
|||
|
|
@ -1,88 +0,0 @@
|
|||
CXX := clang++
|
||||
CC := clang
|
||||
|
||||
BASEDIR = ../..
|
||||
PHONELIBS = ../../phonelibs
|
||||
|
||||
CXXFLAGS := -g -O3 -fPIC -std=c++11 -Wall -Wextra -Wshadow -Weffc++ -Wstrict-aliasing -Werror -MMD
|
||||
|
||||
LDLIBS=-lm -lstdc++ -lrt -lpthread
|
||||
|
||||
UNAME_M := $(shell uname -m)
|
||||
|
||||
YAML_FLAGS = -I$(PHONELIBS)/yaml-cpp/include -I../
|
||||
YAML_LIB = $(abspath $(PHONELIBS)/yaml-cpp/lib/libyaml-cpp.a)
|
||||
|
||||
ifeq ($(UNAME_M),aarch64)
|
||||
LDFLAGS += -llog -lgnustl_shared
|
||||
ZMQ_LIBS = /usr/lib/libzmq.a
|
||||
endif
|
||||
ifeq ($(UNAME_M),x86_64)
|
||||
ZMQ_FLAGS = -I$(BASEDIR)/phonelibs/zmq/x64/include
|
||||
ZMQ_LIBS = $(abspath $(BASEDIR)/phonelibs/zmq/x64/lib/libzmq.a)
|
||||
YAML_DIR = $(PHONELIBS)/yaml-cpp/x64/lib/
|
||||
YAML_LIB = $(abspath $(PHONELIBS)/yaml-cpp/x64/lib/libyaml-cpp.a)
|
||||
endif
|
||||
|
||||
ifdef ASAN
|
||||
CXXFLAGS += -fsanitize=address -fno-omit-frame-pointer
|
||||
LDFLAGS += -fsanitize=address
|
||||
endif
|
||||
|
||||
CXXFLAGS += $(ZMQ_FLAGS) $(YAML_FLAGS)
|
||||
|
||||
OBJS := messaging.o impl_zmq.o impl_msgq.o msgq.o
|
||||
DEPS=$(OBJS:.o=.d)
|
||||
|
||||
TEST_OBJS := test_runner.o msgq_tests.o msgq.o
|
||||
TEST_DEPS=$(TEST_OBJS:.o=.d)
|
||||
|
||||
.PRECIOUS: $(OBJS)
|
||||
.PHONY: all clean test
|
||||
all: bridge messaging.a messaging_pyx.so messaging.so
|
||||
|
||||
test: test_runner
|
||||
./test_runner
|
||||
|
||||
test_runner: $(TEST_OBJS)
|
||||
|
||||
demo: messaging.a demo.o
|
||||
$(CC) $(LDFLAGS) $^ $(LDLIBS) -L. -l:messaging.a -o '$@'
|
||||
|
||||
bridge: messaging.a bridge.o
|
||||
$(CC) $(LDFLAGS) $^ $(LDLIBS) -L. -l:messaging.a -o '$@'
|
||||
|
||||
messaging_pyx.so: messaging.a messaging_pyx_setup.py messaging_pyx.pyx messaging.pxd
|
||||
python3 messaging_pyx_setup.py build_ext --inplace
|
||||
rm -rf build
|
||||
rm -f messaging_pyx.cpp
|
||||
|
||||
messaging.so: $(OBJS)
|
||||
@echo "[ LINK ] $@"
|
||||
mkdir -p libs_so; \
|
||||
cd libs_so; \
|
||||
ar -x $(ZMQ_LIBS); \
|
||||
ar -x $(YAML_LIB);
|
||||
|
||||
$(CXX) -shared $(LDFLAGS) $^ $(LDLIBS) libs_so/*.o -o '$@'
|
||||
chmod 644 '$@'
|
||||
rm -r libs_so
|
||||
|
||||
%.a: $(OBJS)
|
||||
@echo "[ LINK ] $@"
|
||||
mkdir -p libs_a; \
|
||||
cd libs_a; \
|
||||
ar -x $(ZMQ_LIBS); \
|
||||
ar -x $(YAML_LIB);
|
||||
|
||||
ar rcsD '$@' $^ libs_a/*.o
|
||||
rm -r libs_a
|
||||
|
||||
../services.h: ../services.py ../service_list.yaml
|
||||
python3 ../services.py > ../services.h
|
||||
|
||||
clean:
|
||||
@echo "[ CLEAN ]"
|
||||
rm -rf *.so *.a bridge demo libs_a libs_so test_runner $(OBJS) $(DEPS) $(TEST_OBJS) $(TEST_DEPS)
|
||||
|
||||
-include $(DEPS)
|
||||
|
|
@ -1,10 +1,33 @@
|
|||
import os
|
||||
import subprocess
|
||||
import sysconfig
|
||||
from distutils.core import Extension, setup # pylint: disable=import-error,no-name-in-module
|
||||
|
||||
from Cython.Build import cythonize
|
||||
from Cython.Distutils import build_ext
|
||||
|
||||
|
||||
def get_ext_filename_without_platform_suffix(filename):
|
||||
name, ext = os.path.splitext(filename)
|
||||
ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
|
||||
|
||||
if ext_suffix == ext:
|
||||
return filename
|
||||
|
||||
ext_suffix = ext_suffix.replace(ext, '')
|
||||
idx = name.find(ext_suffix)
|
||||
|
||||
if idx == -1:
|
||||
return filename
|
||||
else:
|
||||
return name[:idx] + ext
|
||||
|
||||
|
||||
class BuildExtWithoutPlatformSuffix(build_ext):
|
||||
def get_ext_filename(self, ext_name):
|
||||
filename = super().get_ext_filename(ext_name)
|
||||
return get_ext_filename_without_platform_suffix(filename)
|
||||
|
||||
from common.cython_hacks import BuildExtWithoutPlatformSuffix
|
||||
|
||||
sourcefiles = ['messaging_pyx.pyx']
|
||||
extra_compile_args = ["-std=c++11"]
|
||||
|
|
|
|||
Loading…
Reference in New Issue