From 52c6db8719972a3b29a727eba0b3f045c6390391 Mon Sep 17 00:00:00 2001 From: Willem Melching Date: Wed, 20 Nov 2019 16:32:42 -0800 Subject: [PATCH] Run scons in CI (#14) * try to run scons in azure pipelines * sudo * install capnp * does this run * also clean test runner * remove makefiles * this should build --- .dockerignore | 1 + .gitignore | 1 + Dockerfile | 19 +++++++ Makefile | 62 ---------------------- SConscript | 21 +++++--- SConstruct | 49 ++++++++++++++++++ azure-pipelines.yml | 9 ++-- install_capnp.sh | 7 +-- messaging/.gitignore | 1 + messaging/Makefile | 88 -------------------------------- messaging/messaging_pyx_setup.py | 25 ++++++++- 11 files changed, 117 insertions(+), 166 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile delete mode 100644 Makefile create mode 100644 SConstruct delete mode 100644 messaging/Makefile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5b2d462 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.sconsign.dblite diff --git a/.gitignore b/.gitignore index b697594..f24a0a4 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ libcereal*.a libmessaging.* libmessaging_shared.* services.h +.sconsign.dblite diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f681dce --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +from ubuntu:16.04 + +RUN apt-get update && apt-get install -y libzmq3-dev clang wget git autoconf libtool curl make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python-openssl + +RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash +ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}" +RUN pyenv install 3.7.3 +RUN pyenv global 3.7.3 +RUN pyenv rehash +RUN pip3 install pyyaml==5.1.2 Cython==0.29.14 scons==3.1.1 pycapnp==0.6.4 + +WORKDIR /project/cereal +COPY install_capnp.sh . +RUN ./install_capnp.sh + +ENV PYTHONPATH=/project + +COPY . . +RUN scons -c && scons -j$(nproc) diff --git a/Makefile b/Makefile deleted file mode 100644 index 83318be..0000000 --- a/Makefile +++ /dev/null @@ -1,62 +0,0 @@ -PWD := $(shell pwd) - -SRCS := log.capnp car.capnp - -GENS := gen/cpp/car.capnp.c++ gen/cpp/log.capnp.c++ -JS := gen/js/car.capnp.js gen/js/log.capnp.js - -UNAME_M ?= $(shell uname -m) - -GENS += gen/c/car.capnp.c gen/c/log.capnp.c gen/c/include/c++.capnp.h gen/c/include/java.capnp.h - -ifeq ($(UNAME_M),x86_64) - -ifneq (, $(shell which capnpc-java)) -GENS += gen/java/Car.java gen/java/Log.java -else -$(warning capnpc-java not found, skipping java build) -endif - -endif - - -ifeq ($(UNAME_M),aarch64) -CAPNPC=PATH=$(PWD)/../phonelibs/capnp-cpp/aarch64/bin/:$$PATH capnpc -else -CAPNPC=capnpc -endif - -.PHONY: all -all: $(GENS) -js: $(JS) - -.PHONY: clean -clean: - rm -rf gen - rm -rf node_modules - rm -rf package-lock.json - -gen/c/%.capnp.c: %.capnp - @echo "[ CAPNPC C ] $@" - mkdir -p gen/c/ - $(CAPNPC) '$<' -o c:gen/c/ - -gen/js/%.capnp.js: %.capnp - @echo "[ CAPNPC JavaScript ] $@" - mkdir -p gen/js/ - sh ./generate_javascript.sh - -gen/cpp/%.capnp.c++: %.capnp - @echo "[ CAPNPC C++ ] $@" - mkdir -p gen/cpp/ - $(CAPNPC) '$<' -o c++:gen/cpp/ - -gen/java/Car.java gen/java/Log.java: $(SRCS) - @echo "[ CAPNPC java ] $@" - mkdir -p gen/java/ - $(CAPNPC) $^ -o java:gen/java - -# c-capnproto needs some empty headers -gen/c/include/c++.capnp.h gen/c/include/java.capnp.h: - mkdir -p gen/c/include - touch '$@' diff --git a/SConscript b/SConscript index 7bcb58e..397402f 100644 --- a/SConscript +++ b/SConscript @@ -1,15 +1,18 @@ Import('env', 'arch', 'zmq') +gen_dir = Dir('gen') +messaging_dir = Dir('messaging') + # TODO: remove src-prefix and cereal from command string. can we set working directory? -env.Command(["gen/c/include/c++.capnp.h", "gen/c/include/java.capnp.h"], [], "mkdir -p cereal/gen/c/include && touch $TARGETS") +env.Command(["gen/c/include/c++.capnp.h", "gen/c/include/java.capnp.h"], [], "mkdir -p " + gen_dir.path + "/c/include && touch $TARGETS") env.Command( ['gen/c/car.capnp.c', 'gen/c/log.capnp.c', 'gen/c/car.capnp.h', 'gen/c/log.capnp.h'], ['car.capnp', 'log.capnp'], - 'capnpc $SOURCES --src-prefix=cereal -o c:cereal/gen/c/') + 'capnpc $SOURCES --src-prefix=cereal -o c:' + gen_dir.path + '/c/') env.Command( ['gen/cpp/car.capnp.c++', 'gen/cpp/log.capnp.c++', 'gen/cpp/car.capnp.h', 'gen/cpp/log.capnp.h'], ['car.capnp', 'log.capnp'], - 'capnpc $SOURCES --src-prefix=cereal -o c++:cereal/gen/cpp/') + 'capnpc $SOURCES --src-prefix=cereal -o c++:' + gen_dir.path + '/cpp/') env.Library('cereal', [ 'gen/c/car.capnp.c', @@ -18,10 +21,12 @@ env.Library('cereal', [ 'gen/cpp/log.capnp.c++', ]) + +cereal_dir = Dir('.') env.Command( ['services.h'], ['service_list.yaml', 'services.py'], - 'python3 cereal/services.py > $TARGET') + 'python3 ' + cereal_dir.path + '/services.py > $TARGET') messaging_deps = [ 'messaging/messaging.cc', @@ -39,12 +44,16 @@ if arch == "aarch64": messaging_shared_lib = env.SharedLibrary('messaging_shared', messaging_deps, LIBS=shared_lib_shared_lib) env.Command(['messaging/messaging.so'], [messaging_shared_lib], "chmod 777 $SOURCES && ln -sf `realpath $SOURCES` $TARGET") -env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=['messaging', 'zmq']) +env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=[messaging_lib, 'zmq']) # different target? #env.Program('messaging/demo', ['messaging/demo.cc'], LIBS=['messaging', 'zmq']) + env.Command(['messaging/messaging_pyx.so'], [messaging_lib, 'messaging/messaging_pyx_setup.py', 'messaging/messaging_pyx.pyx', 'messaging/messaging.pxd'], - "cd cereal/messaging && python3 messaging_pyx_setup.py build_ext --inplace") + "cd " + messaging_dir.path + " && python3 messaging_pyx_setup.py build_ext --inplace") + +if GetOption('test'): + env.Program('messaging/test_runner', ['messaging/test_runner.cc', 'messaging/msgq_tests.cc'], LIBS=[messaging_lib]) diff --git a/SConstruct b/SConstruct new file mode 100644 index 0000000..a72286b --- /dev/null +++ b/SConstruct @@ -0,0 +1,49 @@ +import os +import subprocess + +zmq = 'zmq' +arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip() + +cereal_dir = Dir('.') + +cpppath = [ + cereal_dir, + '/usr/lib/include', +] + +AddOption('--test', + action='store_true', + help='build test files') + +AddOption('--asan', + action='store_true', + help='turn on ASAN') + +ccflags_asan = ["-fsanitize=address", "-fno-omit-frame-pointer"] if GetOption('asan') else [] +ldflags_asan = ["-fsanitize=address"] if GetOption('asan') else [] + +env = Environment( + ENV=os.environ, + CC='clang', + CXX='clang++', + CCFLAGS=[ + "-g", + "-fPIC", + "-O2", + "-Werror=implicit-function-declaration", + "-Werror=incompatible-pointer-types", + "-Werror=int-conversion", + "-Werror=return-type", + "-Werror=format-extra-args", + ] + ccflags_asan, + LDFLAGS=ldflags_asan, + LINKFLAGS=ldflags_asan, + + CFLAGS="-std=gnu11", + CXXFLAGS="-std=c++14", + CPPPATH=cpppath, +) + + +Export('env', 'zmq', 'arch') +SConscript(['SConscript']) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0f63d1c..305bc3a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -3,10 +3,7 @@ pool: steps: - script: | - cd messaging - ASAN=1 make test_runner - ./test_runner -r junit -o tests.xml + docker build -t cereal . + docker run cereal bash -c "scons --test --asan -j$(nproc) && messaging/test_runner" + displayName: 'Run Tests' -- task: PublishTestResults@2 - inputs: - testResultsFiles: 'messaging/tests.xml' diff --git a/install_capnp.sh b/install_capnp.sh index b83e1ff..cc570b6 100755 --- a/install_capnp.sh +++ b/install_capnp.sh @@ -8,7 +8,8 @@ tar xvf capnproto-c++-${VERSION}.tar.gz cd capnproto-c++-${VERSION} CXXFLAGS="-fPIC" ./configure -make -j4 +make -j$(nproc) +make install # manually build binaries statically g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -DCAPNP_INCLUDE_DIR=\"/usr/local/include\" -pthread -O2 -DNDEBUG -pthread -pthread -o .libs/capnp src/capnp/compiler/module-loader.o src/capnp/compiler/capnp.o ./.libs/libcapnpc.a ./.libs/libcapnp.a ./.libs/libkj.a -lpthread -pthread @@ -18,7 +19,6 @@ g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -D g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -DCAPNP_INCLUDE_DIR=\"/usr/local/include\" -pthread -O2 -DNDEBUG -pthread -pthread -o .libs/capnpc-capnp src/capnp/compiler/capnpc-capnp.o ./.libs/libcapnp.a ./.libs/libkj.a -lpthread -pthread cp .libs/capnp /usr/local/bin/ -ln -s /usr/local/bin/capnp /usr/local/bin/capnpc cp .libs/capnpc-c++ /usr/local/bin/ cp .libs/capnpc-capnp /usr/local/bin/ cp .libs/*.a /usr/local/lib @@ -30,7 +30,8 @@ cd c-capnproto git submodule update --init --recursive autoreconf -f -i -s CXXFLAGS="-fPIC" ./configure -make -j4 +make -j$(nproc) +make install # manually build binaries statically gcc -fPIC -o .libs/capnpc-c compiler/capnpc-c.o compiler/schema.capnp.o compiler/str.o ./.libs/libcapnp_c.a diff --git a/messaging/.gitignore b/messaging/.gitignore index 0ddca76..dbbe8e2 100644 --- a/messaging/.gitignore +++ b/messaging/.gitignore @@ -7,3 +7,4 @@ test_runner *.a *.so messaging_pyx.cpp +build/ diff --git a/messaging/Makefile b/messaging/Makefile deleted file mode 100644 index fac58b3..0000000 --- a/messaging/Makefile +++ /dev/null @@ -1,88 +0,0 @@ -CXX := clang++ -CC := clang - -BASEDIR = ../.. -PHONELIBS = ../../phonelibs - -CXXFLAGS := -g -O3 -fPIC -std=c++11 -Wall -Wextra -Wshadow -Weffc++ -Wstrict-aliasing -Werror -MMD - -LDLIBS=-lm -lstdc++ -lrt -lpthread - -UNAME_M := $(shell uname -m) - -YAML_FLAGS = -I$(PHONELIBS)/yaml-cpp/include -I../ -YAML_LIB = $(abspath $(PHONELIBS)/yaml-cpp/lib/libyaml-cpp.a) - -ifeq ($(UNAME_M),aarch64) - LDFLAGS += -llog -lgnustl_shared - ZMQ_LIBS = /usr/lib/libzmq.a -endif -ifeq ($(UNAME_M),x86_64) - ZMQ_FLAGS = -I$(BASEDIR)/phonelibs/zmq/x64/include - ZMQ_LIBS = $(abspath $(BASEDIR)/phonelibs/zmq/x64/lib/libzmq.a) - YAML_DIR = $(PHONELIBS)/yaml-cpp/x64/lib/ - YAML_LIB = $(abspath $(PHONELIBS)/yaml-cpp/x64/lib/libyaml-cpp.a) -endif - -ifdef ASAN - CXXFLAGS += -fsanitize=address -fno-omit-frame-pointer - LDFLAGS += -fsanitize=address -endif - -CXXFLAGS += $(ZMQ_FLAGS) $(YAML_FLAGS) - -OBJS := messaging.o impl_zmq.o impl_msgq.o msgq.o -DEPS=$(OBJS:.o=.d) - -TEST_OBJS := test_runner.o msgq_tests.o msgq.o -TEST_DEPS=$(TEST_OBJS:.o=.d) - -.PRECIOUS: $(OBJS) -.PHONY: all clean test -all: bridge messaging.a messaging_pyx.so messaging.so - -test: test_runner - ./test_runner - -test_runner: $(TEST_OBJS) - -demo: messaging.a demo.o - $(CC) $(LDFLAGS) $^ $(LDLIBS) -L. -l:messaging.a -o '$@' - -bridge: messaging.a bridge.o - $(CC) $(LDFLAGS) $^ $(LDLIBS) -L. -l:messaging.a -o '$@' - -messaging_pyx.so: messaging.a messaging_pyx_setup.py messaging_pyx.pyx messaging.pxd - python3 messaging_pyx_setup.py build_ext --inplace - rm -rf build - rm -f messaging_pyx.cpp - -messaging.so: $(OBJS) - @echo "[ LINK ] $@" - mkdir -p libs_so; \ - cd libs_so; \ - ar -x $(ZMQ_LIBS); \ - ar -x $(YAML_LIB); - - $(CXX) -shared $(LDFLAGS) $^ $(LDLIBS) libs_so/*.o -o '$@' - chmod 644 '$@' - rm -r libs_so - -%.a: $(OBJS) - @echo "[ LINK ] $@" - mkdir -p libs_a; \ - cd libs_a; \ - ar -x $(ZMQ_LIBS); \ - ar -x $(YAML_LIB); - - ar rcsD '$@' $^ libs_a/*.o - rm -r libs_a - -../services.h: ../services.py ../service_list.yaml - python3 ../services.py > ../services.h - -clean: - @echo "[ CLEAN ]" - rm -rf *.so *.a bridge demo libs_a libs_so test_runner $(OBJS) $(DEPS) $(TEST_OBJS) $(TEST_DEPS) - --include $(DEPS) diff --git a/messaging/messaging_pyx_setup.py b/messaging/messaging_pyx_setup.py index bfc0da4..a763d89 100644 --- a/messaging/messaging_pyx_setup.py +++ b/messaging/messaging_pyx_setup.py @@ -1,10 +1,33 @@ import os import subprocess +import sysconfig from distutils.core import Extension, setup # pylint: disable=import-error,no-name-in-module from Cython.Build import cythonize +from Cython.Distutils import build_ext + + +def get_ext_filename_without_platform_suffix(filename): + name, ext = os.path.splitext(filename) + ext_suffix = sysconfig.get_config_var('EXT_SUFFIX') + + if ext_suffix == ext: + return filename + + ext_suffix = ext_suffix.replace(ext, '') + idx = name.find(ext_suffix) + + if idx == -1: + return filename + else: + return name[:idx] + ext + + +class BuildExtWithoutPlatformSuffix(build_ext): + def get_ext_filename(self, ext_name): + filename = super().get_ext_filename(ext_name) + return get_ext_filename_without_platform_suffix(filename) -from common.cython_hacks import BuildExtWithoutPlatformSuffix sourcefiles = ['messaging_pyx.pyx'] extra_compile_args = ["-std=c++11"]