Run scons in CI (#14)

* try to run scons in azure pipelines

* sudo

* install capnp

* does this run

* also clean test runner

* remove makefiles

* this should build

.dockerignore

@@ -0,0 +1 @@
+.sconsign.dblite

.gitignore

@@ -9,4 +9,5 @@ libcereal*.a
 libmessaging.*
 libmessaging_shared.*
 services.h
+.sconsign.dblite
 

Dockerfile

@@ -0,0 +1,19 @@
+from ubuntu:16.04
+
+RUN apt-get update && apt-get install -y libzmq3-dev clang wget git autoconf libtool curl make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python-openssl
+
+RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
+ENV PATH="/root/.pyenv/bin:/root/.pyenv/shims:${PATH}"
+RUN pyenv install 3.7.3
+RUN pyenv global 3.7.3
+RUN pyenv rehash
+RUN pip3 install pyyaml==5.1.2 Cython==0.29.14 scons==3.1.1 pycapnp==0.6.4
+
+WORKDIR /project/cereal
+COPY install_capnp.sh .
+RUN ./install_capnp.sh
+
+ENV PYTHONPATH=/project
+
+COPY . .
+RUN scons -c && scons -j$(nproc)

Makefile

@@ -1,62 +0,0 @@
-PWD := $(shell pwd)
-
-SRCS := log.capnp car.capnp
-
-GENS := gen/cpp/car.capnp.c++ gen/cpp/log.capnp.c++
-JS := gen/js/car.capnp.js gen/js/log.capnp.js
-
-UNAME_M ?= $(shell uname -m)
-
-GENS += gen/c/car.capnp.c gen/c/log.capnp.c gen/c/include/c++.capnp.h gen/c/include/java.capnp.h
-
-ifeq ($(UNAME_M),x86_64)
-
-ifneq (, $(shell which capnpc-java))
-GENS += gen/java/Car.java gen/java/Log.java
-else
-$(warning capnpc-java not found, skipping java build)
-endif
-
-endif
-
-
-ifeq ($(UNAME_M),aarch64)
-CAPNPC=PATH=$(PWD)/../phonelibs/capnp-cpp/aarch64/bin/:$$PATH capnpc
-else
-CAPNPC=capnpc
-endif
-
-.PHONY: all
-all: $(GENS)
-js: $(JS)
-
-.PHONY: clean
-clean:
-	rm -rf gen
-	rm -rf node_modules
-	rm -rf package-lock.json
-
-gen/c/%.capnp.c: %.capnp
-	@echo "[ CAPNPC C ] $@"
-	mkdir -p gen/c/
-	$(CAPNPC) '$<' -o c:gen/c/
-
-gen/js/%.capnp.js: %.capnp
-	@echo "[ CAPNPC JavaScript ] $@"
-	mkdir -p gen/js/
-	sh ./generate_javascript.sh
-
-gen/cpp/%.capnp.c++: %.capnp
-	@echo "[ CAPNPC C++ ] $@"
-	mkdir -p gen/cpp/
-	$(CAPNPC) '$<' -o c++:gen/cpp/
-
-gen/java/Car.java gen/java/Log.java: $(SRCS)
-	@echo "[ CAPNPC java ] $@"
-	mkdir -p gen/java/
-	$(CAPNPC) $^ -o java:gen/java
-
-# c-capnproto needs some empty headers
-gen/c/include/c++.capnp.h gen/c/include/java.capnp.h:
-	mkdir -p gen/c/include
-	touch '$@'

SConscript

@@ -1,15 +1,18 @@
 Import('env', 'arch', 'zmq')
 
+gen_dir = Dir('gen')
+messaging_dir = Dir('messaging')
+
 # TODO: remove src-prefix and cereal from command string. can we set working directory?
-env.Command(["gen/c/include/c++.capnp.h", "gen/c/include/java.capnp.h"], [], "mkdir -p cereal/gen/c/include && touch $TARGETS")
+env.Command(["gen/c/include/c++.capnp.h", "gen/c/include/java.capnp.h"], [], "mkdir -p " + gen_dir.path + "/c/include && touch $TARGETS")
 env.Command(
   ['gen/c/car.capnp.c', 'gen/c/log.capnp.c', 'gen/c/car.capnp.h', 'gen/c/log.capnp.h'],
   ['car.capnp', 'log.capnp'],
-  'capnpc $SOURCES --src-prefix=cereal -o c:cereal/gen/c/')
+  'capnpc $SOURCES --src-prefix=cereal -o c:' + gen_dir.path + '/c/')
 env.Command(
   ['gen/cpp/car.capnp.c++', 'gen/cpp/log.capnp.c++', 'gen/cpp/car.capnp.h', 'gen/cpp/log.capnp.h'],
   ['car.capnp', 'log.capnp'],
-  'capnpc $SOURCES --src-prefix=cereal -o c++:cereal/gen/cpp/')
+  'capnpc $SOURCES --src-prefix=cereal -o c++:' + gen_dir.path + '/cpp/')
 
 env.Library('cereal', [
     'gen/c/car.capnp.c',
@@ -18,10 +21,12 @@ env.Library('cereal', [
     'gen/cpp/log.capnp.c++',
   ])
 
+
+cereal_dir = Dir('.')
 env.Command(
   ['services.h'],
   ['service_list.yaml', 'services.py'],
-  'python3 cereal/services.py > $TARGET')
+  'python3 ' + cereal_dir.path + '/services.py > $TARGET')
 
 messaging_deps = [
   'messaging/messaging.cc',
@@ -39,12 +44,16 @@ if arch == "aarch64":
 messaging_shared_lib = env.SharedLibrary('messaging_shared', messaging_deps, LIBS=shared_lib_shared_lib)
 env.Command(['messaging/messaging.so'], [messaging_shared_lib], "chmod 777 $SOURCES && ln -sf `realpath $SOURCES` $TARGET")
 
-env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=['messaging', 'zmq'])
+env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=[messaging_lib, 'zmq'])
 
 # different target?
 #env.Program('messaging/demo', ['messaging/demo.cc'], LIBS=['messaging', 'zmq'])
 
+
 env.Command(['messaging/messaging_pyx.so'],
   [messaging_lib, 'messaging/messaging_pyx_setup.py', 'messaging/messaging_pyx.pyx', 'messaging/messaging.pxd'],
-  "cd cereal/messaging && python3 messaging_pyx_setup.py build_ext --inplace")
+  "cd " + messaging_dir.path + " && python3 messaging_pyx_setup.py build_ext --inplace")
 
+
+if GetOption('test'):
+  env.Program('messaging/test_runner', ['messaging/test_runner.cc', 'messaging/msgq_tests.cc'], LIBS=[messaging_lib])

SConstruct

@@ -0,0 +1,49 @@
+import os
+import subprocess
+
+zmq = 'zmq'
+arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
+
+cereal_dir = Dir('.')
+
+cpppath = [
+    cereal_dir,
+    '/usr/lib/include',
+]
+
+AddOption('--test',
+          action='store_true',
+          help='build test files')
+
+AddOption('--asan',
+          action='store_true',
+          help='turn on ASAN')
+
+ccflags_asan = ["-fsanitize=address", "-fno-omit-frame-pointer"] if GetOption('asan') else []
+ldflags_asan = ["-fsanitize=address"] if GetOption('asan') else []
+
+env = Environment(
+  ENV=os.environ,
+  CC='clang',
+  CXX='clang++',
+  CCFLAGS=[
+    "-g",
+    "-fPIC",
+    "-O2",
+    "-Werror=implicit-function-declaration",
+    "-Werror=incompatible-pointer-types",
+    "-Werror=int-conversion",
+    "-Werror=return-type",
+    "-Werror=format-extra-args",
+  ] + ccflags_asan,
+  LDFLAGS=ldflags_asan,
+  LINKFLAGS=ldflags_asan,
+
+  CFLAGS="-std=gnu11",
+  CXXFLAGS="-std=c++14",
+  CPPPATH=cpppath,
+)
+
+
+Export('env', 'zmq', 'arch')
+SConscript(['SConscript'])

azure-pipelines.yml

@@ -3,10 +3,7 @@ pool:
 
 steps:
 - script: |
-    cd messaging
+    docker build -t cereal .
-    ASAN=1 make test_runner
+    docker run cereal bash -c "scons --test --asan -j$(nproc) && messaging/test_runner"
-    ./test_runner -r junit -o tests.xml
+
   displayName: 'Run Tests'
-- task: PublishTestResults@2
-  inputs:
-    testResultsFiles: 'messaging/tests.xml'

install_capnp.sh

@@ -8,7 +8,8 @@ tar xvf capnproto-c++-${VERSION}.tar.gz
 cd capnproto-c++-${VERSION}
 CXXFLAGS="-fPIC" ./configure
 
-make -j4
+make -j$(nproc)
+make install
 
 # manually build binaries statically
 g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -DCAPNP_INCLUDE_DIR=\"/usr/local/include\" -pthread -O2 -DNDEBUG -pthread -pthread -o .libs/capnp src/capnp/compiler/module-loader.o src/capnp/compiler/capnp.o  ./.libs/libcapnpc.a ./.libs/libcapnp.a ./.libs/libkj.a -lpthread -pthread
@@ -18,7 +19,6 @@ g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -D
 g++ -std=gnu++11 -I./src -I./src -DKJ_HEADER_WARNINGS -DCAPNP_HEADER_WARNINGS -DCAPNP_INCLUDE_DIR=\"/usr/local/include\" -pthread -O2 -DNDEBUG -pthread -pthread -o .libs/capnpc-capnp src/capnp/compiler/capnpc-capnp.o  ./.libs/libcapnp.a ./.libs/libkj.a -lpthread -pthread
 
 cp .libs/capnp /usr/local/bin/
-ln -s /usr/local/bin/capnp /usr/local/bin/capnpc
 cp .libs/capnpc-c++ /usr/local/bin/
 cp .libs/capnpc-capnp /usr/local/bin/
 cp .libs/*.a /usr/local/lib
@@ -30,7 +30,8 @@ cd c-capnproto
 git submodule update --init --recursive
 autoreconf -f -i -s
 CXXFLAGS="-fPIC" ./configure
-make -j4
+make -j$(nproc)
+make install
 
 # manually build binaries statically
 gcc -fPIC -o .libs/capnpc-c compiler/capnpc-c.o compiler/schema.capnp.o compiler/str.o  ./.libs/libcapnp_c.a

messaging/.gitignore

@@ -7,3 +7,4 @@ test_runner
 *.a
 *.so
 messaging_pyx.cpp
+build/

messaging/Makefile

@@ -1,88 +0,0 @@
-CXX := clang++
-CC := clang
-
-BASEDIR = ../..
-PHONELIBS = ../../phonelibs
-
-CXXFLAGS := -g -O3 -fPIC -std=c++11 -Wall -Wextra -Wshadow -Weffc++ -Wstrict-aliasing -Werror -MMD
-
-LDLIBS=-lm -lstdc++ -lrt -lpthread
-
-UNAME_M := $(shell uname -m)
-
-YAML_FLAGS = -I$(PHONELIBS)/yaml-cpp/include -I../
-YAML_LIB = $(abspath $(PHONELIBS)/yaml-cpp/lib/libyaml-cpp.a)
-
-ifeq ($(UNAME_M),aarch64)
-	LDFLAGS += -llog -lgnustl_shared
-	ZMQ_LIBS = /usr/lib/libzmq.a
-endif
-ifeq ($(UNAME_M),x86_64)
-	ZMQ_FLAGS = -I$(BASEDIR)/phonelibs/zmq/x64/include
-	ZMQ_LIBS = $(abspath $(BASEDIR)/phonelibs/zmq/x64/lib/libzmq.a)
-	YAML_DIR = $(PHONELIBS)/yaml-cpp/x64/lib/
-	YAML_LIB = $(abspath $(PHONELIBS)/yaml-cpp/x64/lib/libyaml-cpp.a)
-endif
-
-ifdef ASAN
-	CXXFLAGS += -fsanitize=address -fno-omit-frame-pointer
-  LDFLAGS += -fsanitize=address
-endif
-
-CXXFLAGS += $(ZMQ_FLAGS) $(YAML_FLAGS)
-
-OBJS := messaging.o impl_zmq.o impl_msgq.o msgq.o
-DEPS=$(OBJS:.o=.d)
-
-TEST_OBJS := test_runner.o msgq_tests.o msgq.o
-TEST_DEPS=$(TEST_OBJS:.o=.d)
-
-.PRECIOUS: $(OBJS)
-.PHONY: all clean test
-all: bridge messaging.a messaging_pyx.so messaging.so
-
-test: test_runner
-	./test_runner
-
-test_runner: $(TEST_OBJS)
-
-demo: messaging.a demo.o
-	$(CC) $(LDFLAGS) $^ $(LDLIBS) -L. -l:messaging.a -o '$@'
-
-bridge: messaging.a bridge.o
-	$(CC) $(LDFLAGS) $^ $(LDLIBS) -L. -l:messaging.a -o '$@'
-
-messaging_pyx.so: messaging.a messaging_pyx_setup.py messaging_pyx.pyx messaging.pxd
-	python3 messaging_pyx_setup.py build_ext --inplace
-	rm -rf build
-	rm -f messaging_pyx.cpp
-
-messaging.so: $(OBJS)
-	@echo "[ LINK ] $@"
-	mkdir -p libs_so; \
-		cd libs_so; \
-		ar -x $(ZMQ_LIBS); \
-		ar -x $(YAML_LIB);
-
-	$(CXX) -shared $(LDFLAGS) $^ $(LDLIBS) libs_so/*.o  -o '$@'
-	chmod 644 '$@'
-	rm -r libs_so
-
-%.a: $(OBJS)
-	@echo "[ LINK ] $@"
-	mkdir -p libs_a; \
-		cd libs_a; \
-		ar -x $(ZMQ_LIBS); \
-		ar -x $(YAML_LIB);
-
-	ar rcsD '$@' $^ libs_a/*.o
-	rm -r libs_a
-
-../services.h: ../services.py ../service_list.yaml
-	python3 ../services.py > ../services.h
-
-clean:
-	@echo "[ CLEAN ]"
-	rm -rf *.so *.a bridge demo libs_a libs_so test_runner $(OBJS) $(DEPS) $(TEST_OBJS) $(TEST_DEPS)
-
--include $(DEPS)

messaging/messaging_pyx_setup.py

@@ -1,10 +1,33 @@
 import os
 import subprocess
+import sysconfig
 from distutils.core import Extension, setup  # pylint: disable=import-error,no-name-in-module
 
 from Cython.Build import cythonize
+from Cython.Distutils import build_ext
+
+
+def get_ext_filename_without_platform_suffix(filename):
+  name, ext = os.path.splitext(filename)
+  ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
+
+  if ext_suffix == ext:
+    return filename
+
+  ext_suffix = ext_suffix.replace(ext, '')
+  idx = name.find(ext_suffix)
+
+  if idx == -1:
+    return filename
+  else:
+    return name[:idx] + ext
+
+
+class BuildExtWithoutPlatformSuffix(build_ext):
+  def get_ext_filename(self, ext_name):
+    filename = super().get_ext_filename(ext_name)
+    return get_ext_filename_without_platform_suffix(filename)
 
-from common.cython_hacks import BuildExtWithoutPlatformSuffix
 
 sourcefiles = ['messaging_pyx.pyx']
 extra_compile_args = ["-std=c++11"]