Scons cython builder (#316)

* migrate to home pc

* style

* this works

* this is hacky

* why doesn't this work

* this should work

* no lint

* link against libdbc wtih abs path

* new env

Co-authored-by: Adeeb Shihadeh <adeebshihadeh@gmail.com>

.gitignore

@@ -1,6 +1,7 @@
 .mypy_cache/
 *.pyc
 *.os
+*.o
 *.tmp
 *.dylib
 .*.swp

.pre-commit-config.yaml

@@ -14,6 +14,7 @@ repos:
     rev: master
     hooks:
     -   id: flake8
+        exclude: 'site_scons/'
         args:
         - --ignore=E111,E114,E121,E124,E302,E501,E741
         - --statistics
@@ -24,6 +25,7 @@ repos:
         entry: pylint
         language: system
         types: [python]
+        exclude: 'site_scons/'
         args:
         - --disable=C,R,W0613,W0511,W0212,W0201,W0311,W0106,W0603,W0621,W0703,E1136
 -   repo: local

Dockerfile

@@ -46,6 +46,7 @@ WORKDIR /project
 RUN git clone https://github.com/commaai/cereal.git /project/cereal
 
 COPY SConstruct .
+COPY ./site_scons /project/site_scons
 COPY . /project/opendbc
 
 RUN rm -rf /project/opendbc/.git

SConstruct

@@ -1,24 +1,20 @@
-import Cython
-import distutils
 import os
 import subprocess
-import sys
+import sysconfig
 
 zmq = 'zmq'
 arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
 
-# Rebuild cython extensions if python, distutils, or cython change
-cython_dependencies = [Value(v) for v in (sys.version, distutils.__version__, Cython.__version__)]
-Export('cython_dependencies')
-
 cereal_dir = Dir('.')
 
+python_path = sysconfig.get_paths()['include']
 cpppath = [
-    '#',
-    '#cereal',
-    "#cereal/messaging",
-    "#opendbc/can",
-    '/usr/lib/include',
+  '#',
+  '#cereal',
+  "#cereal/messaging",
+  "#opendbc/can",
+  '/usr/lib/include',
+  python_path
 ]
 
 AddOption('--test',
@@ -45,10 +41,14 @@ env = Environment(
   ] + ccflags_asan,
   LDFLAGS=ldflags_asan,
   LINKFLAGS=ldflags_asan,
-
+  LIBPATH=[
+    "#opendbc/can/",
+  ],
   CFLAGS="-std=gnu11",
   CXXFLAGS="-std=c++1z",
   CPPPATH=cpppath,
+  CYTHONCFILESUFFIX=".cpp",
+  tools=["default", "cython"]
 )
 
 Export('env', 'zmq', 'arch')
@@ -57,5 +57,24 @@ cereal = [File('#cereal/libcereal.a')]
 messaging = [File('#cereal/libmessaging.a')]
 Export('cereal', 'messaging')
 
+
+envCython = env.Clone()
+envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-deprecated-declarations"]
+
+python_libs = []
+if arch == "Darwin":
+  envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"]
+elif arch == "aarch64":
+  envCython["LINKFLAGS"] = ["-shared"]
+
+  python_libs.append(os.path.basename(python_path))
+else:
+  envCython["LINKFLAGS"] = ["-pthread", "-shared"]
+
+envCython["LIBS"] = python_libs
+
+Export('envCython')
+
+
 SConscript(['cereal/SConscript'])
 SConscript(['opendbc/can/SConscript'])

can/SConscript

@@ -1,4 +1,4 @@
-Import('env', 'cereal', 'cython_dependencies')
+Import('env', 'envCython', 'cereal')
 
 import os
 from opendbc.can.process_dbc import process
@@ -13,10 +13,10 @@ for x in sorted(os.listdir('../')):
     dbc = env.Command(out_fn, in_fn, compile_dbc)
     dbcs.append(dbc)
 
-
 libdbc = env.SharedLibrary('libdbc', ["dbc.cc", "parser.cc", "packer.cc", "common.cc"]+dbcs, LIBS=["capnp", "kj"])
 
 # Build packer and parser
-env.Command(['packer_pyx.so', 'packer_pyx.cpp', 'parser_pyx.so', 'parser_pyx.cpp'],
-            cython_dependencies + [libdbc, cereal, 'common_pyx_setup.py', 'common.pxd', 'packer_pyx.pyx', 'parser_pyx.pyx', 'packer.cc', 'parser.cc'],
-            "cd opendbc/can && python3 common_pyx_setup.py build_ext --inplace")
+lenv = envCython.Clone()
+lenv["LINKFLAGS"] += [libdbc[0].get_labspath()]
+lenv.Program('parser_pyx.so', 'parser_pyx.pyx')
+lenv.Program('packer_pyx.so', 'packer_pyx.pyx')

can/common_pyx_setup.py

@@ -1,94 +0,0 @@
-import os
-import subprocess
-import sysconfig
-import platform
-from distutils.core import Extension, setup  # pylint: disable=import-error,no-name-in-module
-
-from Cython.Build import cythonize
-from Cython.Distutils import build_ext
-
-
-ANNOTATE = os.getenv('ANNOTATE') is not None
-BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../"))
-
-
-def get_ext_filename_without_platform_suffix(filename):
-  name, ext = os.path.splitext(filename)
-  ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
-
-  if ext_suffix == ext:
-    return filename
-
-  ext_suffix = ext_suffix.replace(ext, '')
-  idx = name.find(ext_suffix)
-
-  if idx == -1:
-    return filename
-  else:
-    return name[:idx] + ext
-
-
-class BuildExtWithoutPlatformSuffix(build_ext):
-  def get_ext_filename(self, ext_name):
-    filename = super().get_ext_filename(ext_name)
-    return get_ext_filename_without_platform_suffix(filename)
-
-
-extra_compile_args = ["-std=c++1z", "-Wno-nullability-completeness"]
-ARCH = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()  # pylint: disable=unexpected-keyword-arg
-if ARCH == "aarch64":
-  extra_compile_args += ["-Wno-deprecated-register"]
-
-if platform.system() == "Darwin":
-  libdbc = "libdbc.dylib"
-else:
-  libdbc = "libdbc.so"
-
-extra_link_args = [os.path.join(BASEDIR, 'opendbc', 'can', libdbc)]
-include_dirs = [
-  BASEDIR,
-  os.path.join(BASEDIR, 'phonelibs'),
-]
-
-# Build CAN Parser
-
-setup(name='CAN parser',
-      cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
-      ext_modules=cythonize(
-        Extension(
-          "parser_pyx",
-          language="c++",
-          sources=['parser_pyx.pyx'],
-          extra_compile_args=extra_compile_args,
-          include_dirs=include_dirs,
-          extra_link_args=extra_link_args,
-        ),
-        nthreads=4,
-        annotate=ANNOTATE
-      ),
-)
-
-if platform.system() == "Darwin":
-  os.system("install_name_tool -change opendbc/can/libdbc.dylib " + BASEDIR + "/opendbc/can/libdbc.dylib parser_pyx.so")
-
-
-# Build CAN Packer
-
-setup(name='CAN packer',
-      cmdclass={'build_ext': BuildExtWithoutPlatformSuffix},
-      ext_modules=cythonize(
-        Extension(
-          "packer_pyx",
-          language="c++",
-          sources=['packer_pyx.pyx'],
-          extra_compile_args=extra_compile_args,
-          include_dirs=include_dirs,
-          extra_link_args=extra_link_args,
-        ),
-        nthreads=4,
-        annotate=ANNOTATE
-      ),
-)
-
-if platform.system() == "Darwin":
-  os.system("install_name_tool -change opendbc/can/libdbc.dylib " + BASEDIR + "/opendbc/can/libdbc.dylib packer_pyx.so")

can/packer_pyx.pyx

@@ -8,8 +8,8 @@ from libcpp.string cimport string
 from libcpp cimport bool
 from posix.dlfcn cimport dlopen, dlsym, RTLD_LAZY
 
-from common cimport CANPacker as cpp_CANPacker
-from common cimport dbc_lookup, SignalPackValue, DBC
+from .common cimport CANPacker as cpp_CANPacker
+from .common cimport dbc_lookup, SignalPackValue, DBC
 
 
 cdef class CANPacker:
@@ -22,8 +22,8 @@ cdef class CANPacker:
   def __init__(self, dbc_name):
     self.dbc = dbc_lookup(dbc_name)
     if not self.dbc:
-      raise RuntimeError("Can't lookup" + dbc_name)
-      
+      raise RuntimeError(f"Can't lookup {dbc_name}")
+
     self.packer = new cpp_CANPacker(dbc_name)
     num_msgs = self.dbc[0].num_msgs
     for i in range(num_msgs):

can/parser_pyx.pyx

@@ -8,8 +8,8 @@ from libc.stdint cimport uint32_t, uint64_t, uint16_t
 from libcpp.map cimport map
 from libcpp cimport bool
 
-from common cimport CANParser as cpp_CANParser
-from common cimport SignalParseOptions, MessageParseOptions, dbc_lookup, SignalValue, DBC
+from .common cimport CANParser as cpp_CANParser
+from .common cimport SignalParseOptions, MessageParseOptions, dbc_lookup, SignalValue, DBC
 
 import os
 import numbers

site_scons/site_tools/cython.py

@@ -0,0 +1,38 @@
+import SCons
+from SCons.Action import Action
+
+cythonAction = Action("$CYTHONCOM")
+
+def create_builder(env):
+  try:
+    cython = env['BUILDERS']['Cython']
+  except KeyError:
+    cython = SCons.Builder.Builder(
+               action = cythonAction,
+               emitter = {},
+               suffix = cython_suffix_emitter,
+               single_source = 1
+             )
+    env['BUILDERS']['Cython'] = cython
+  return cython
+
+def cython_suffix_emitter(env, source):
+  return "$CYTHONCFILESUFFIX"
+
+def generate(env):
+  env["CYTHON"] = "cythonize"
+  env["CYTHONCOM"] = "$CYTHON $CYTHONFLAGS $SOURCE"
+  env["CYTHONCFILESUFFIX"] = ".cpp"
+
+  c_file, _ = SCons.Tool.createCFileBuilders(env)
+
+  c_file.suffix['.pyx'] = cython_suffix_emitter
+  c_file.add_action('.pyx', cythonAction)
+
+  c_file.suffix['.py'] = cython_suffix_emitter
+  c_file.add_action('.py', cythonAction)
+
+  create_builder(env)
+
+def exists(env):
+  return True