build system cleanups (#36202)

* it's all common

* never getting fixed

* it's just tici

* reorders

* qcom2 -> tici

* Revert "qcom2 -> tici"

This reverts commit f4d849b2952cb0e662975805db6a1d32511ed392.

* Reapply "qcom2 -> tici"

This reverts commit 58b193cb8de872830f8a7821a339edca14e4a337.

* is tici

* lil more

* Revert "is tici"

This reverts commit a169be18d3fdcb3ef8317a63a89d8becadabfad8.

* Revert "Reapply "qcom2 -> tici""

This reverts commit 26f9c0e7d068fc8a1a5f07383b3616e619cd4e8c.

* qcom2 -> __tici__

* lil more

* mv lenv

* clean that up

* lil more]

* fix

* lil more

SConstruct

@@ -3,144 +3,52 @@ import subprocess
 import sys
 import sysconfig
 import platform
+import shlex
 import numpy as np
 
 import SCons.Errors
 
 SCons.Warnings.warningAsException(True)
 
-# pending upstream fix - https://github.com/SCons/scons/issues/4461
-#SetOption('warn', 'all')
-
-TICI = os.path.isfile('/TICI')
-AGNOS = TICI
-
 Decider('MD5-timestamp')
 
 SetOption('num_jobs', max(1, int(os.cpu_count()/2)))
 
-AddOption('--kaitai',
-          action='store_true',
-          help='Regenerate kaitai struct parsers')
-
-AddOption('--asan',
-          action='store_true',
-          help='turn on ASAN')
-
-AddOption('--ubsan',
-          action='store_true',
-          help='turn on UBSan')
-
-AddOption('--ccflags',
-          action='store',
-          type='string',
-          default='',
-          help='pass arbitrary flags over the command line')
-
-AddOption('--mutation',
-          action='store_true',
-          help='generate mutation-ready code')
-
+AddOption('--kaitai', action='store_true', help='Regenerate kaitai struct parsers')
+AddOption('--asan', action='store_true', help='turn on ASAN')
+AddOption('--ubsan', action='store_true', help='turn on UBSan')
+AddOption('--mutation', action='store_true', help='generate mutation-ready code')
+AddOption('--ccflags', action='store', type='string', default='', help='pass arbitrary flags over the command line')
 AddOption('--minimal',
           action='store_false',
           dest='extras',
           default=os.path.exists(File('#.lfsconfig').abspath), # minimal by default on release branch (where there's no LFS)
           help='the minimum build to run openpilot. no tests, tools, etc.')
 
-## Architecture name breakdown (arch)
-## - larch64: linux tici aarch64
-## - aarch64: linux pc aarch64
-## - x86_64:  linux pc x64
-## - Darwin:  mac x64 or arm64
-real_arch = arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
+# Detect platform
+arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
 if platform.system() == "Darwin":
   arch = "Darwin"
   brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip()
-elif arch == "aarch64" and AGNOS:
+elif arch == "aarch64" and os.path.isfile('/TICI'):
   arch = "larch64"
-assert arch in ["larch64", "aarch64", "x86_64", "Darwin"]
-
-lenv = {
-  "PATH": os.environ['PATH'],
-  "PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath,
-
-  "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
-  "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
-  "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
-}
-
-rpath = []
-
-if arch == "larch64":
-  cpppath = [
-    "#third_party/opencl/include",
-  ]
-
-  libpath = [
-    "/usr/local/lib",
-    "/system/vendor/lib64",
-    f"#third_party/acados/{arch}/lib",
-  ]
-
-  libpath += [
-    "#third_party/libyuv/larch64/lib",
-    "/usr/lib/aarch64-linux-gnu"
-  ]
-  cflags = ["-DQCOM2", "-mcpu=cortex-a57"]
-  cxxflags = ["-DQCOM2", "-mcpu=cortex-a57"]
-  rpath += ["/usr/local/lib"]
-else:
-  cflags = []
-  cxxflags = []
-  cpppath = []
-  rpath += []
-
-  # MacOS
-  if arch == "Darwin":
-    libpath = [
-      f"#third_party/libyuv/{arch}/lib",
-      f"#third_party/acados/{arch}/lib",
-      f"{brew_prefix}/lib",
-      f"{brew_prefix}/opt/openssl@3.0/lib",
-      f"{brew_prefix}/opt/llvm/lib/c++",
-      "/System/Library/Frameworks/OpenGL.framework/Libraries",
-    ]
-
-    cflags += ["-DGL_SILENCE_DEPRECATION"]
-    cxxflags += ["-DGL_SILENCE_DEPRECATION"]
-    cpppath += [
-      f"{brew_prefix}/include",
-      f"{brew_prefix}/opt/openssl@3.0/include",
-    ]
-  # Linux
-  else:
-    libpath = [
-      f"#third_party/acados/{arch}/lib",
-      f"#third_party/libyuv/{arch}/lib",
-      "/usr/lib",
-      "/usr/local/lib",
-    ]
-
-if GetOption('asan'):
-  ccflags = ["-fsanitize=address", "-fno-omit-frame-pointer"]
-  ldflags = ["-fsanitize=address"]
-elif GetOption('ubsan'):
-  ccflags = ["-fsanitize=undefined"]
-  ldflags = ["-fsanitize=undefined"]
-else:
-  ccflags = []
-  ldflags = []
-
-# no --as-needed on mac linker
-if arch != "Darwin":
-  ldflags += ["-Wl,--as-needed", "-Wl,--no-undefined"]
-
-ccflags_option = GetOption('ccflags')
-if ccflags_option:
-  ccflags += ccflags_option.split(' ')
+assert arch in [
+  "larch64",  # linux tici arm64
+  "aarch64",  # linux pc arm64
+  "x86_64",   # linux pc x64
+  "Darwin",   # macOS arm64 (x86 not supported)
+]
 
 env = Environment(
-  ENV=lenv,
+  ENV={
+    "PATH": os.environ['PATH'],
+    "PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath,
+    "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
+    "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
+    "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
+  },
+  CC='clang',
+  CXX='clang++',
   CCFLAGS=[
     "-g",
     "-fPIC",
@@ -153,36 +61,31 @@ env = Environment(
     "-Wno-c99-designator",
     "-Wno-reorder-init-list",
     "-Wno-vla-cxx-extension",
-  ] + cflags + ccflags,
-
-  CPPPATH=cpppath + [
+  ],
+  CFLAGS=["-std=gnu11"],
+  CXXFLAGS=["-std=c++1z"],
+  CPPPATH=[
     "#",
+    "#msgq",
+    "#third_party",
+    "#third_party/json11",
+    "#third_party/linux/include",
     "#third_party/acados/include",
     "#third_party/acados/include/blasfeo/include",
     "#third_party/acados/include/hpipm/include",
     "#third_party/catch2/include",
     "#third_party/libyuv/include",
-    "#third_party/json11",
-    "#third_party/linux/include",
-    "#third_party",
-    "#msgq",
   ],
-
-  CC='clang',
-  CXX='clang++',
-  LINKFLAGS=ldflags,
-
-  RPATH=rpath,
-
-  CFLAGS=["-std=gnu11"] + cflags,
-  CXXFLAGS=["-std=c++1z"] + cxxflags,
-  LIBPATH=libpath + [
+  LIBPATH=[
+    "#common",
     "#msgq_repo",
     "#third_party",
     "#selfdrive/pandad",
-    "#common",
     "#rednose/helpers",
+    f"#third_party/libyuv/{arch}/lib",
+    f"#third_party/acados/{arch}/lib",
   ],
+  RPATH=[],
   CYTHONCFILESUFFIX=".cpp",
   COMPILATIONDB_USE_ABSPATH=True,
   REDNOSE_ROOT="#",
@@ -190,29 +93,63 @@ env = Environment(
   toolpath=["#site_scons/site_tools", "#rednose_repo/site_scons/site_tools"],
 )
 
-if arch == "Darwin":
-  # RPATH is not supported on macOS, instead use the linker flags
-  darwin_rpath_link_flags = [f"-Wl,-rpath,{path}" for path in env["RPATH"]]
-  env["LINKFLAGS"] += darwin_rpath_link_flags
+# Arch-specific flags and paths
+if arch == "larch64":
+  env.Append(CPPPATH=["#third_party/opencl/include"])
+  env.Append(LIBPATH=[
+    "/usr/local/lib",
+    "/system/vendor/lib64",
+    "/usr/lib/aarch64-linux-gnu",
+  ])
+  arch_flags = ["-D__TICI__", "-mcpu=cortex-a57"]
+  env.Append(CCFLAGS=arch_flags)
+  env.Append(CXXFLAGS=arch_flags)
+elif arch == "Darwin":
+  env.Append(LIBPATH=[
+    f"{brew_prefix}/lib",
+    f"{brew_prefix}/opt/openssl@3.0/lib",
+    f"{brew_prefix}/opt/llvm/lib/c++",
+    "/System/Library/Frameworks/OpenGL.framework/Libraries",
+  ])
+  env.Append(CCFLAGS=["-DGL_SILENCE_DEPRECATION"])
+  env.Append(CXXFLAGS=["-DGL_SILENCE_DEPRECATION"])
+  env.Append(CPPPATH=[
+    f"{brew_prefix}/include",
+    f"{brew_prefix}/opt/openssl@3.0/include",
+  ])
+else:
+  env.Append(LIBPATH=[
+    "/usr/lib",
+    "/usr/local/lib",
+  ])
 
-env.CompilationDatabase('compile_commands.json')
+# Sanitizers and extra CCFLAGS from CLI
+if GetOption('asan'):
+  env.Append(CCFLAGS=["-fsanitize=address", "-fno-omit-frame-pointer"])
+  env.Append(LINKFLAGS=["-fsanitize=address"])
+elif GetOption('ubsan'):
+  env.Append(CCFLAGS=["-fsanitize=undefined"])
+  env.Append(LINKFLAGS=["-fsanitize=undefined"])
 
-# Setup cache dir
-cache_dir = '/data/scons_cache' if AGNOS else '/tmp/scons_cache'
-CacheDir(cache_dir)
-Clean(["."], cache_dir)
+_extra_cc = shlex.split(GetOption('ccflags') or '')
+if _extra_cc:
+  env.Append(CCFLAGS=_extra_cc)
 
+# no --as-needed on mac linker
+if arch != "Darwin":
+  env.Append(LINKFLAGS=["-Wl,--as-needed", "-Wl,--no-undefined"])
+
+# progress output
 node_interval = 5
 node_count = 0
 def progress_function(node):
   global node_count
   node_count += node_interval
   sys.stderr.write("progress: %d\n" % node_count)
-
 if os.environ.get('SCONS_PROGRESS'):
   Progress(progress_function, interval=node_interval)
 
-# Cython build environment
+# ********** Cython build environment **********
 py_include = sysconfig.get_paths()['include']
 envCython = env.Clone()
 envCython["CPPPATH"] += [py_include, np.get_include()]
@@ -221,14 +158,14 @@ envCython["CCFLAGS"].remove("-Werror")
 
 envCython["LIBS"] = []
 if arch == "Darwin":
-  envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"] + darwin_rpath_link_flags
+  envCython["LINKFLAGS"] = env["LINKFLAGS"] + ["-bundle", "-undefined", "dynamic_lookup"]
 else:
   envCython["LINKFLAGS"] = ["-pthread", "-shared"]
 
 np_version = SCons.Script.Value(np.__version__)
 Export('envCython', 'np_version')
 
-# Qt build environment
+# ********** Qt build environment **********
 qt_env = env.Clone()
 qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "DBus", "Xml"]
 
@@ -278,16 +215,20 @@ qt_env['CXXFLAGS'] += qt_flags
 qt_env['LIBPATH'] += ['#selfdrive/ui', ]
 qt_env['LIBS'] = qt_libs
 
-Export('env', 'qt_env', 'arch', 'real_arch')
+Export('env', 'qt_env', 'arch')
+
+# Setup cache dir
+cache_dir = '/data/scons_cache' if arch == "larch64" else '/tmp/scons_cache'
+CacheDir(cache_dir)
+Clean(["."], cache_dir)
+
+# ********** start building stuff **********
 
 # Build common module
 SConscript(['common/SConscript'])
-Import('_common', '_gpucommon')
-
+Import('_common')
 common = [_common, 'json11', 'zmq']
-gpucommon = [_gpucommon]
-
-Export('common', 'gpucommon')
+Export('common')
 
 # Build messaging (cereal + msgq + socketmaster + their dependencies)
 # Enable swaglog include in submodules
@@ -327,3 +268,6 @@ if Dir('#tools/cabana/').exists() and GetOption('extras'):
   SConscript(['tools/replay/SConscript'])
   if arch != "larch64":
     SConscript(['tools/cabana/SConscript'])
+
+
+env.CompilationDatabase('compile_commands.json')

common/SConscript

@@ -5,17 +5,12 @@ common_libs = [
   'swaglog.cc',
   'util.cc',
   'watchdog.cc',
-  'ratekeeper.cc'
-]
-
-_common = env.Library('common', common_libs, LIBS="json11")
-
-files = [
+  'ratekeeper.cc',
   'clutil.cc',
 ]
 
-_gpucommon = env.Library('gpucommon', files)
-Export('_common', '_gpucommon')
+_common = env.Library('common', common_libs, LIBS="json11")
+Export('_common')
 
 if GetOption('extras'):
   env.Program('tests/test_common',

selfdrive/modeld/SConscript

@@ -1,11 +1,11 @@
 import os
 import glob
 
-Import('env', 'envCython', 'arch', 'cereal', 'messaging', 'common', 'gpucommon', 'visionipc', 'transformations')
+Import('env', 'envCython', 'arch', 'cereal', 'messaging', 'common', 'visionipc', 'transformations')
 lenv = env.Clone()
 lenvCython = envCython.Clone()
 
-libs = [cereal, messaging, visionipc, gpucommon, common, 'capnp', 'kj', 'pthread']
+libs = [cereal, messaging, visionipc, common, 'capnp', 'kj', 'pthread']
 frameworks = []
 
 common_src = [

selfdrive/ui/qt/qt_window.cc

@@ -13,7 +13,7 @@ void setMainWindow(QWidget *w) {
   }
   w->show();
 
-#ifdef QCOM2
+#ifdef __TICI__
   QPlatformNativeInterface *native = QGuiApplication::platformNativeInterface();
   wl_surface *s = reinterpret_cast<wl_surface*>(native->nativeResourceForWindow("surface", w->windowHandle()));
   wl_surface_set_buffer_transform(s, WL_OUTPUT_TRANSFORM_270);

selfdrive/ui/qt/qt_window.h

@@ -6,7 +6,7 @@
 #include <QScreen>
 #include <QWidget>
 
-#ifdef QCOM2
+#ifdef __TICI__
 #include <qpa/qplatformnativeinterface.h>
 #include <wayland-client-protocol.h>
 #include <QPlatformSurfaceEvent>

selfdrive/ui/qt/widgets/cameraview.cc

@@ -27,7 +27,7 @@ const char frame_vertex_shader[] =
   "}\n";
 
 const char frame_fragment_shader[] =
-#ifdef QCOM2
+#ifdef __TICI__
   "#version 300 es\n"
   "#extension GL_OES_EGL_image_external_essl3 : enable\n"
   "precision mediump float;\n"
@@ -79,7 +79,7 @@ CameraWidget::~CameraWidget() {
     glDeleteVertexArrays(1, &frame_vao);
     glDeleteBuffers(1, &frame_vbo);
     glDeleteBuffers(1, &frame_ibo);
-#ifndef QCOM2
+#ifndef __TICI__
     glDeleteTextures(2, textures);
 #endif
   }
@@ -137,7 +137,7 @@ void CameraWidget::initializeGL() {
 
   glUseProgram(program->programId());
 
-#ifdef QCOM2
+#ifdef __TICI__
   glUniform1i(program->uniformLocation("uTexture"), 0);
 #else
   glGenTextures(2, textures);
@@ -165,7 +165,7 @@ void CameraWidget::stopVipcThread() {
     vipc_thread = nullptr;
   }
 
-#ifdef QCOM2
+#ifdef __TICI__
   EGLDisplay egl_display = eglGetCurrentDisplay();
   assert(egl_display != EGL_NO_DISPLAY);
   for (auto &pair : egl_images) {
@@ -226,7 +226,7 @@ void CameraWidget::paintGL() {
   glUseProgram(program->programId());
   glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
 
-#ifdef QCOM2
+#ifdef __TICI__
   // no frame copy
   glActiveTexture(GL_TEXTURE0);
   glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, egl_images[frame->idx]);
@@ -263,7 +263,7 @@ void CameraWidget::vipcConnected(VisionIpcClient *vipc_client) {
   stream_height = vipc_client->buffers[0].height;
   stream_stride = vipc_client->buffers[0].stride;
 
-#ifdef QCOM2
+#ifdef __TICI__
   EGLDisplay egl_display = eglGetCurrentDisplay();
   assert(egl_display != EGL_NO_DISPLAY);
   for (auto &pair : egl_images) {

selfdrive/ui/qt/widgets/cameraview.h

@@ -13,7 +13,7 @@
 #include <QOpenGLWidget>
 #include <QThread>
 
-#ifdef QCOM2
+#ifdef __TICI__
 #define EGL_EGLEXT_PROTOTYPES
 #define EGL_NO_X11
 #define GL_TEXTURE_EXTERNAL_OES 0x8D65
@@ -63,7 +63,7 @@ protected:
   std::unique_ptr<QOpenGLShaderProgram> program;
   QColor bg = QColor("#000000");
 
-#ifdef QCOM2
+#ifdef __TICI__
   std::map<int, EGLImageKHR> egl_images;
 #endif
 

system/camerad/SConscript

@@ -1,6 +1,6 @@
-Import('env', 'arch', 'messaging', 'common', 'gpucommon', 'visionipc')
+Import('env', 'arch', 'messaging', 'common', 'visionipc')
 
-libs = [common, 'OpenCL', messaging, visionipc, gpucommon]
+libs = [common, 'OpenCL', messaging, visionipc]
 
 if arch != "Darwin":
   camera_obj = env.Object(['cameras/camera_qcom2.cc', 'cameras/camera_common.cc', 'cameras/spectra.cc',

system/camerad/cameras/camera_qcom2.cc

@@ -12,7 +12,7 @@
 #include <string>
 #include <vector>
 
-#ifdef QCOM2
+#ifdef __TICI__
 #include "CL/cl_ext_qcom.h"
 #else
 #define CL_PRIORITY_HINT_HIGH_QCOM NULL

system/hardware/hw.h

@@ -5,7 +5,7 @@
 #include "system/hardware/base.h"
 #include "common/util.h"
 
-#if QCOM2
+#if __TICI__
 #include "system/hardware/tici/hardware.h"
 #define Hardware HardwareTici
 #else

system/loggerd/encoderd.cc

@@ -3,7 +3,7 @@
 #include "system/loggerd/loggerd.h"
 #include "system/loggerd/encoder/jpeg_encoder.h"
 
-#ifdef QCOM2
+#ifdef __TICI__
 #include "system/loggerd/encoder/v4l_encoder.h"
 #define Encoder V4LEncoder
 #else