build nightly casync build in jenkins (#31880)

* casync in jenkins

* rename some stuff, add a readme

* slightly better names

* clean

* more cleanup

* cleaner

* release3 staging too

* always rm the signed version

* cleanups

* in build dir

* better name

* simpler

* more

* divider

* built

* build

* and contains

* add channel description

* and git branches

* and build required

* move this up

* these are terms

* updates

* 3/3x

* bullets

* wording

* version metadata

* git type

* more channel -> release

* more build

* just release

* more channel to release

* also fix jenkins

* use build_metadata

* fix normailzed

* also normalized

* and here

* use build_metadata

* dont commit that

* don't touch the git stuff

* branch

* don't need that

* or that

* improved names

* build_metadata

* use this instead

* fix

* build

* test nightly build again

* fix

* fixes

* Revert "test nightly build again"

This reverts commit be5e7aa7089bfc0947c9b2b484d0277c109ee089.

Jenkinsfile

@@ -142,6 +142,23 @@ def setupCredentials() {
 }
 
 
+def build_release(String channel_name) {
+  return parallel (
+    "${channel_name} (git)": {
+      deviceStage("build git", "tici-needs-can", [], [
+        ["build ${channel_name}", "RELEASE_BRANCH=${channel_name} $SOURCE_DIR/release/build_release.sh"],
+      ])
+    },
+    "${channel_name} (casync)": {
+      deviceStage("build casync", "tici-needs-can", [], [
+        ["build ${channel_name}", "RELEASE=1 OPENPILOT_CHANNEL=${channel_name} BUILD_DIR=/data/openpilot_build CASYNC_DIR=/data/casync $SOURCE_DIR/release/create_casync_build.sh"],
+        //["upload ${channel_name}", "OPENPILOT_CHANNEL=${channel_name} $SOURCE_DIR/release/upload_casync_release.sh"],
+      ])
+    }
+  )
+}
+
+
 node {
   env.CI = "1"
   env.PYTHONWARNINGS = "error"
@@ -164,15 +181,11 @@ node {
 
   try {
     if (env.BRANCH_NAME == 'devel-staging') {
-      deviceStage("build release3-staging", "tici-needs-can", [], [
-        ["build release3-staging", "RELEASE_BRANCH=release3-staging $SOURCE_DIR/release/build_release.sh"],
-      ])
+      build_release("release3-staging")
     }
 
     if (env.BRANCH_NAME == 'master-ci') {
-      deviceStage("build nightly", "tici-needs-can", [], [
-        ["build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"],
-      ])
+      build_release("nightly")
     }
 
     if (!env.BRANCH_NAME.matches(excludeRegex)) {

release/README.md

@@ -0,0 +1,44 @@
+# openpilot releases
+
+
+## terms
+
+- `channel` - a named version of openpilot (git branch, casync caidx) which receives updates
+- `build` - a release which is already built for the comma 3/3x and contains only required files for running openpilot and identifying the release
+
+- `build_style` - type of build, either `debug` or `release`
+  - `debug` - build with `ALLOW_DEBUG=true`, can test experimental features like longitudinal on alpha cars
+  - `release` - build with `ALLOW_DEBUG=false`, experimental features disabled
+
+
+## openpilot channels
+
+| channel      | build_style | description                                                       |
+| -----------  | ----------- | ----------                                                        |
+| release      | `release`   | stable release of openpilot                                       |
+| staging      | `release`   | release candidate of openpilot for final verification             |
+| nightly      | `release`   | generated nightly from last commit passing CI tests               |
+| master       | `debug`     | current master commit with experimental features enabled          |
+| git branches | `debug`     | installed manually, experimental features enabled, build required |
+
+
+## creating casync build
+
+`create_casync_build.sh` - creates a casync openpilot build, ready to upload to `openpilot-releases`
+
+```bash
+# run on a tici, within the directory you want to create the build from.
+# creates a prebuilt version of openpilot into BUILD_DIR and outputs the caidx
+# and other casync files into CASYNC_DIR for uploading to openpilot-releases.
+BUILD_DIR=/data/openpilot_build    \
+CASYNC_DIR=/data/casync            \
+OPENPILOT_CHANNEL=nightly          \
+release/create_casync_build.sh
+```
+
+`upload_casync_release.sh` - helper for uploading a casync build to `openpilot-releases`
+
+
+## release builds
+
+to create a release build, set `RELEASE=1` environment variable when running the build script

release/copy_build_files.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+SOURCE_DIR=$1
+TARGET_DIR=$2
+
+if [ -f /TICI ]; then
+  FILES_SRC="release/files_tici"
+else
+  echo "no release files set"
+  exit 1
+fi
+
+cd $SOURCE_DIR
+cp -pR --parents $(cat release/files_common) $BUILD_DIR/
+cp -pR --parents $(cat $FILES_SRC) $TARGET_DIR/

release/create_casync_build.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/bash
+
+set -ex
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+
+CASYNC_DIR="${CASYNC_DIR:=/tmp/casync}"
+SOURCE_DIR="$(git -C $DIR rev-parse --show-toplevel)"
+BUILD_DIR="${BUILD_DIR:=$(mktemp -d)}"
+
+echo "Creating casync release from $SOURCE_DIR to $CASYNC_DIR"
+
+mkdir -p $CASYNC_DIR
+rm -rf $BUILD_DIR
+mkdir -p $BUILD_DIR
+
+release/copy_build_files.sh $SOURCE_DIR $BUILD_DIR
+release/create_prebuilt.sh $BUILD_DIR
+
+cd $SOURCE_DIR
+release/create_casync_release.py $BUILD_DIR $CASYNC_DIR $OPENPILOT_CHANNEL

release/create_casync_release.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+
+import argparse
+import pathlib
+
+from openpilot.system.updated.casync.common import create_caexclude_file, create_casync_release, create_build_metadata_file
+from openpilot.system.version import get_build_metadata
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description="creates a casync release")
+  parser.add_argument("target_dir", type=str, help="target directory to build channel from")
+  parser.add_argument("output_dir", type=str, help="output directory for the channel")
+  parser.add_argument("channel", type=str, help="what channel this build is")
+  args = parser.parse_args()
+
+  target_dir = pathlib.Path(args.target_dir)
+  output_dir = pathlib.Path(args.output_dir)
+
+  create_build_metadata_file(target_dir, get_build_metadata(), args.channel)
+  create_caexclude_file(target_dir)
+
+  digest, caidx = create_casync_release(target_dir, output_dir, args.channel)
+
+  print(f"Created casync release from {target_dir} to {caidx} with digest {digest}")

release/create_prebuilt.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/bash -e
+
+# runs on tici to create a prebuilt version of a release
+
+set -ex
+
+BUILD_DIR=$1
+
+cd $BUILD_DIR
+
+# Build
+export PYTHONPATH="$BUILD_DIR"
+
+rm -f panda/board/obj/panda.bin.signed
+rm -f panda/board/obj/panda_h7.bin.signed
+
+if [ -n "$RELEASE" ]; then
+  export CERT=/data/pandaextra/certs/release
+fi
+
+scons -j$(nproc)
+
+# Cleanup
+find . -name '*.a' -delete
+find . -name '*.o' -delete
+find . -name '*.os' -delete
+find . -name '*.pyc' -delete
+find . -name 'moc_*' -delete
+find . -name '__pycache__' -delete
+rm -rf .sconsign.dblite Jenkinsfile release/
+rm selfdrive/modeld/models/supercombo.onnx
+
+# Mark as prebuilt release
+touch prebuilt

release/upload_casync_release.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+CASYNC_DIR="${CASYNC_DIR:=/tmp/casync}"
+
+OPENPILOT_RELEASES="https://commadist.blob.core.windows.net/openpilot-releases/"
+
+SAS="$(python -c 'from tools.lib.azure_container import get_container_sas;print(get_container_sas("commadist","openpilot-releases"))')"
+
+azcopy cp "$CASYNC_DIR*" "$OPENPILOT_RELEASES?$SAS" --recursive

system/updated/casync/common.py

@@ -0,0 +1,53 @@
+import dataclasses
+import json
+import pathlib
+import subprocess
+
+from openpilot.system.version import BUILD_METADATA_FILENAME, BuildMetadata
+
+
+CASYNC_ARGS = ["--with=symlinks", "--with=permissions"]
+CASYNC_FILES = [BUILD_METADATA_FILENAME, ".caexclude"]
+
+
+def run(cmd):
+  return subprocess.check_output(cmd)
+
+
+def get_exclude_set(path) -> set[str]:
+  exclude_set = set(CASYNC_FILES)
+
+  for file in path.rglob("*"):
+    if file.is_file() or file.is_symlink():
+
+      while file.resolve() != path.resolve():
+        exclude_set.add(str(file.relative_to(path)))
+
+        file = file.parent
+
+  return exclude_set
+
+
+def create_caexclude_file(path: pathlib.Path):
+  with open(path / ".caexclude", "w") as f:
+    # exclude everything except the paths already in the release
+    f.write("*\n")
+    f.write(".*\n")
+
+    for file in sorted(get_exclude_set(path)):
+      f.write(f"!{file}\n")
+
+
+def create_build_metadata_file(path: pathlib.Path, build_metadata: BuildMetadata, channel: str):
+  with open(path / BUILD_METADATA_FILENAME, "w") as f:
+    build_metadata_dict = dataclasses.asdict(build_metadata)
+    build_metadata_dict["channel"] = channel
+    build_metadata_dict["openpilot"].pop("is_dirty")  # this is determined at runtime
+    f.write(json.dumps(build_metadata_dict))
+
+
+def create_casync_release(target_dir: pathlib.Path, output_dir: pathlib.Path, channel: str):
+  caidx_file = output_dir / f"{channel}.caidx"
+  run(["casync", "make", *CASYNC_ARGS, caidx_file, target_dir])
+  digest = run(["casync", "digest", *CASYNC_ARGS, target_dir]).decode("utf-8").strip()
+  return digest, caidx_file