diff --git a/Jenkinsfile b/Jenkinsfile index 83bd9b5d8..1a0e2c73e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -63,6 +63,7 @@ if [ -f /TICI ]; then rm -rf /tmp/tmp* rm -rf ~/.commacache rm -rf /dev/shm/* + rm -rf /dev/tmp/tmp* if ! systemctl is-active --quiet systemd-resolved; then echo "restarting resolved" @@ -179,12 +180,14 @@ def build_git_release(String channel_name) { def build_casync_release(String channel_name, def is_release) { - def extra_env = is_release ? "RELEASE=1" : "" + def extra_env = is_release ? "RELEASE=1 " : "" + def build_dir = "/data/openpilot" + + extra_env += "TMPDIR=/data/tmp PYTHONPATH=$SOURCE_DIR" return deviceStage("build casync", "tici-needs-can", [], [ - ["build", "${extra_env} BUILD_DIR=/data/openpilot CASYNC_DIR=/data/casync/openpilot $SOURCE_DIR/release/create_casync_build.sh"], - ["create manifest", "$SOURCE_DIR/release/create_release_manifest.py /data/openpilot /data/manifest.json && cat /data/manifest.json"], - ["upload and cleanup", "PYTHONWARNINGS=ignore $SOURCE_DIR/release/upload_casync_release.py /data/casync && rm -rf /data/casync"], + ["build", "${extra_env} $SOURCE_DIR/release/build_release.sh ${build_dir}"], + ["package + upload", "${extra_env} $SOURCE_DIR/release/package_casync_build.py ${build_dir}"], ]) } @@ -199,8 +202,7 @@ def build_stage() { }, 'publish agnos': { pcStage("publish agnos") { - sh "release/create_casync_agnos_release.py /tmp/casync/agnos /tmp/casync_tmp" - sh "PYTHONWARNINGS=ignore ${env.WORKSPACE}/release/upload_casync_release.py /tmp/casync" + sh "PYTHONWARNINGS=ignore release/package_casync_agnos.py" } } ) diff --git a/SConstruct b/SConstruct index 50dbda4b8..4db1392a7 100644 --- a/SConstruct +++ b/SConstruct @@ -376,11 +376,13 @@ SConscript([ ]) if arch != "Darwin": SConscript([ - 'system/camerad/SConscript', 'system/sensord/SConscript', 'system/logcatd/SConscript', ]) +if arch == "larch64": + SConscript(['system/camerad/SConscript']) + # Build openpilot SConscript(['third_party/SConscript']) diff --git a/release/README.md b/release/README.md index eaf2d2b53..89e6cce6f 100644 --- a/release/README.md +++ b/release/README.md @@ -4,8 +4,7 @@ ## terms - `channel` - a named version of openpilot (git branch, casync caibx) which receives updates -- `build` - a release which is already built for the comma 3/3x and contains only required files for running openpilot and identifying the release - +- `build` - a copy of openpilot ready for distribution, already built for a specific device - `build_style` - type of build, either `debug` or `release` - `debug` - build with `ALLOW_DEBUG=true`, can test experimental features like longitudinal on alpha cars - `release` - build with `ALLOW_DEBUG=false`, experimental features disabled @@ -22,21 +21,13 @@ | git branches | `debug` | installed manually, experimental features enabled, build required | -## creating casync build +## build -`create_casync_build.sh` - creates a casync openpilot build, ready to upload to `openpilot-releases` +`release/build_release.sh ` - creates an openpilot build into `build_dir`, ready for distribution -```bash -# run on a tici, within the directory you want to create the build from. -# creates a prebuilt version of openpilot into BUILD_DIR and outputs the caibx -# of a tarball containing the full prebuilt openpilot release -BUILD_DIR=/data/openpilot_build \ -CASYNC_DIR=/data/casync \ -release/create_casync_build.sh -``` - -`upload_casync_release.sh` - helper for uploading a casync build to `openpilot-releases` +## packaging a casync release +`release/package_casync_build.py ` - packages an openpilot build into a casync tar and uploads to `openpilot-releases` ## release builds diff --git a/release/build_release.sh b/release/build_release.sh new file mode 100755 index 000000000..19c06700f --- /dev/null +++ b/release/build_release.sh @@ -0,0 +1,51 @@ +#!/usr/bin/bash + +set -e + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" +SOURCE_DIR="$(git -C $DIR rev-parse --show-toplevel)" +BUILD_DIR=${1:-$(mktemp -d)} + +if [ -f /TICI ]; then + FILES_SRC="release/files_tici" +else + FILES_SRC="release/files_pc" +fi + +echo "Building openpilot into $BUILD_DIR" + +rm -rf $BUILD_DIR +mkdir -p $BUILD_DIR + +# Copy required files to BUILD_DIR +cd $SOURCE_DIR +cp -pR --parents $(cat release/files_common) $BUILD_DIR/ +cp -pR --parents $(cat $FILES_SRC) $BUILD_DIR/ + +# Build + cleanup +cd $BUILD_DIR +export PYTHONPATH="$BUILD_DIR" + +rm -f panda/board/obj/panda.bin.signed +rm -f panda/board/obj/panda_h7.bin.signed + +if [ -n "$RELEASE" ]; then + export CERT=/data/pandaextra/certs/release +fi + +scons -j$(nproc) + +# Cleanup +find . -name '*.a' -delete +find . -name '*.o' -delete +find . -name '*.os' -delete +find . -name '*.pyc' -delete +find . -name 'moc_*' -delete +find . -name '__pycache__' -delete +rm -rf .sconsign.dblite Jenkinsfile release/ +rm selfdrive/modeld/models/supercombo.onnx + +# Mark as prebuilt release +touch prebuilt + +echo "----- openpilot has been built to $BUILD_DIR -----" diff --git a/release/copy_build_files.sh b/release/copy_build_files.sh deleted file mode 100755 index b40bd4b76..000000000 --- a/release/copy_build_files.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -SOURCE_DIR=$1 -TARGET_DIR=$2 - -if [ -f /TICI ]; then - FILES_SRC="release/files_tici" -else - echo "no release files set" - exit 1 -fi - -cd $SOURCE_DIR -cp -pR --parents $(cat release/files_common) $TARGET_DIR/ -cp -pR --parents $(cat $FILES_SRC) $TARGET_DIR/ diff --git a/release/create_casync_build.sh b/release/create_casync_build.sh deleted file mode 100755 index 256bad4b4..000000000 --- a/release/create_casync_build.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/bash - -set -ex - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" - -CASYNC_DIR="${CASYNC_DIR:=/tmp/casync}" -SOURCE_DIR="$(git -C $DIR rev-parse --show-toplevel)" -BUILD_DIR="${BUILD_DIR:=$(mktemp -d)}" -PYTHONPATH="$SOURCE_DIR" - -echo "Creating casync release from $SOURCE_DIR to $CASYNC_DIR" - -cd $SOURCE_DIR -mkdir -p $CASYNC_DIR -rm -rf $BUILD_DIR -mkdir -p $BUILD_DIR - -release/copy_build_files.sh $SOURCE_DIR $BUILD_DIR -release/create_prebuilt.sh $BUILD_DIR - -cd $SOURCE_DIR -release/create_casync_release.py $BUILD_DIR $CASYNC_DIR diff --git a/release/create_casync_release.py b/release/create_casync_release.py deleted file mode 100755 index 11629f3ab..000000000 --- a/release/create_casync_release.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python - -import argparse -import os -import pathlib - -from openpilot.system.updated.casync.common import create_casync_release, create_build_metadata_file -from openpilot.system.version import get_build_metadata - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="creates a casync release") - parser.add_argument("target_dir", type=str, help="target directory to build channel from") - parser.add_argument("output_dir", type=str, help="output directory for the channel") - args = parser.parse_args() - - target_dir = pathlib.Path(args.target_dir) - output_dir = pathlib.Path(args.output_dir) - - build_metadata = get_build_metadata() - build_metadata.openpilot.build_style = "release" if os.environ.get("RELEASE", None) is not None else "debug" - - create_build_metadata_file(target_dir, build_metadata) - - digest, caibx = create_casync_release(target_dir, output_dir, build_metadata.canonical) - - print(f"Created casync release from {target_dir} to {caibx} with digest {digest}") diff --git a/release/create_prebuilt.sh b/release/create_prebuilt.sh deleted file mode 100755 index 6d3768c53..000000000 --- a/release/create_prebuilt.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/bash -e - -# runs on tici to create a prebuilt version of a release - -set -ex - -BUILD_DIR=$1 - -cd $BUILD_DIR - -# Build -export PYTHONPATH="$BUILD_DIR" - -rm -f panda/board/obj/panda.bin.signed -rm -f panda/board/obj/panda_h7.bin.signed - -if [ -n "$RELEASE" ]; then - export CERT=/data/pandaextra/certs/release -fi - -scons -j$(nproc) - -# Cleanup -find . -name '*.a' -delete -find . -name '*.o' -delete -find . -name '*.os' -delete -find . -name '*.pyc' -delete -find . -name 'moc_*' -delete -find . -name '__pycache__' -delete -rm -rf .sconsign.dblite Jenkinsfile release/ -rm selfdrive/modeld/models/supercombo.onnx - -# Mark as prebuilt release -touch prebuilt diff --git a/release/create_release_manifest.py b/release/create_release_manifest.py deleted file mode 100755 index 1d235f10f..000000000 --- a/release/create_release_manifest.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import dataclasses -import json -import os -import pathlib - -from openpilot.system.hardware.tici.agnos import AGNOS_MANIFEST_FILE, get_partition_path -from openpilot.system.version import get_build_metadata - - -BASE_URL = "https://commadist.blob.core.windows.net" - -OPENPILOT_RELEASES = f"{BASE_URL}/openpilot-releases/openpilot" -AGNOS_RELEASES = f"{BASE_URL}/openpilot-releases/agnos" - - -def create_partition_manifest(partition): - agnos_filename = os.path.basename(partition["url"]).split(".")[0] - - return { - "type": "partition", - "casync": { - "caibx": f"{AGNOS_RELEASES}/{agnos_filename}.caibx" - }, - "path": get_partition_path(0, partition), - "ab": True, - "size": partition["size"], - "full_check": partition["full_check"], - "hash_raw": partition["hash_raw"], - } - - -def create_openpilot_manifest(build_metadata): - return { - "type": "path_tarred", - "path": "/data/openpilot", - "casync": { - "caibx": f"{OPENPILOT_RELEASES}/{build_metadata.canonical}.caibx" - } - } - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="creates a casync release") - parser.add_argument("target_dir", type=str, help="directory of the channel to create manifest from") - parser.add_argument("output_file", type=str, help="output file to put the manifest") - args = parser.parse_args() - - with open(pathlib.Path(args.target_dir) / AGNOS_MANIFEST_FILE) as f: - agnos_manifest = json.load(f) - - build_metadata = get_build_metadata(args.target_dir) - - ret = { - "build_metadata": dataclasses.asdict(build_metadata), - "manifest": [ - *[create_partition_manifest(entry) for entry in agnos_manifest], - create_openpilot_manifest(build_metadata) - ] - } - - with open(args.output_file, "w") as f: - f.write(json.dumps(ret, indent=2)) diff --git a/release/create_casync_agnos_release.py b/release/package_casync_agnos.py similarity index 72% rename from release/create_casync_agnos_release.py rename to release/package_casync_agnos.py index 1b55d0fcc..b80cdfa98 100755 --- a/release/create_casync_agnos_release.py +++ b/release/package_casync_agnos.py @@ -8,27 +8,27 @@ import time from openpilot.common.basedir import BASEDIR from openpilot.system.hardware.tici.agnos import StreamingDecompressor, unsparsify, noop, AGNOS_MANIFEST_FILE from openpilot.system.updated.casync.common import create_casync_from_file +from release.package_casync_build import upload_casync_release if __name__ == "__main__": parser = argparse.ArgumentParser(description="creates a casync release") - parser.add_argument("output_dir", type=str, help="output directory for the channel") - parser.add_argument("working_dir", type=str, help="working directory") parser.add_argument("--manifest", type=str, help="json manifest to create agnos release from", \ default=str(pathlib.Path(BASEDIR) / AGNOS_MANIFEST_FILE)) args = parser.parse_args() - output_dir = pathlib.Path(args.output_dir) - output_dir.mkdir(parents=True, exist_ok=True) - - working_dir = pathlib.Path(args.working_dir) - working_dir.mkdir(parents=True, exist_ok=True) - manifest_file = pathlib.Path(args.manifest) - with tempfile.NamedTemporaryFile(dir=str(working_dir)) as entry_file: - entry_path = pathlib.Path(entry_file.name) + with tempfile.TemporaryDirectory() as temp_dir: + working_dir = pathlib.Path(temp_dir) + casync_dir = working_dir / "casync" + casync_dir.mkdir() + + agnos_casync_dir = casync_dir / "agnos" + agnos_casync_dir.mkdir() + + entry_path = working_dir / "entry" with open(manifest_file) as f: manifest = json.load(f) @@ -53,5 +53,7 @@ if __name__ == "__main__": start = time.monotonic() agnos_filename = os.path.basename(entry["url"]).split(".")[0] - create_casync_from_file(entry_path, output_dir, agnos_filename) + create_casync_from_file(entry_path, agnos_casync_dir, agnos_filename) print(f"created casnc in {time.monotonic() - start}") + + upload_casync_release(casync_dir) diff --git a/release/package_casync_build.py b/release/package_casync_build.py new file mode 100755 index 000000000..5f92e893b --- /dev/null +++ b/release/package_casync_build.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 + +# packages a casync release, uploads to azure, and creates a manifest + +import argparse +import dataclasses +import json +import os +import pathlib +import tempfile + +from openpilot.system.hardware.tici.agnos import AGNOS_MANIFEST_FILE, get_partition_path +from openpilot.system.updated.casync.common import create_build_metadata_file, create_casync_release +from openpilot.system.version import get_build_metadata +from openpilot.tools.lib.azure_container import AzureContainer + + +BASE_URL = "https://commadist.blob.core.windows.net" + +OPENPILOT_RELEASES = f"{BASE_URL}/openpilot-releases/openpilot" +AGNOS_RELEASES = f"{BASE_URL}/openpilot-releases/agnos" + + +def create_casync_caibx(target_dir: pathlib.Path, output_dir: pathlib.Path): + output_dir.mkdir() + build_metadata = get_build_metadata() + build_metadata.openpilot.build_style = "release" if os.environ.get("RELEASE", None) is not None else "debug" + + create_build_metadata_file(target_dir, build_metadata) + + digest, caibx = create_casync_release(target_dir, output_dir, build_metadata.canonical) + + print(f"Created casync release from {target_dir} to {caibx} with digest {digest}") + + +def upload_casync_release(casync_dir: pathlib.Path): + if "AZURE_TOKEN_OPENPILOT_RELEASES" in os.environ: + os.environ["AZURE_TOKEN"] = os.environ["AZURE_TOKEN_OPENPILOT_RELEASES"] + + OPENPILOT_RELEASES_CONTAINER = AzureContainer("commadist", "openpilot-releases") + + for f in casync_dir.rglob("*"): + if f.is_file(): + blob_name = f.relative_to(casync_dir) + print(f"uploading {f} to {blob_name}") + OPENPILOT_RELEASES_CONTAINER.upload_file(str(f), str(blob_name), overwrite=True) + + +def create_partition_manifest(partition): + agnos_filename = os.path.basename(partition["url"]).split(".")[0] + + return { + "type": "partition", + "casync": { + "caibx": f"{AGNOS_RELEASES}/{agnos_filename}.caibx" + }, + "path": get_partition_path(0, partition), + "ab": True, + "size": partition["size"], + "full_check": partition["full_check"], + "hash_raw": partition["hash_raw"], + } + + +def create_openpilot_manifest(build_metadata): + return { + "type": "path_tarred", + "path": "/data/openpilot", + "casync": { + "caibx": f"{OPENPILOT_RELEASES}/{build_metadata.canonical}.caibx" + } + } + + +def create_manifest(target_dir): + with open(pathlib.Path(target_dir) / AGNOS_MANIFEST_FILE) as f: + agnos_manifest = json.load(f) + + build_metadata = get_build_metadata(args.target_dir) + + return { + "build_metadata": dataclasses.asdict(build_metadata), + "manifest": [ + *[create_partition_manifest(entry) for entry in agnos_manifest], + create_openpilot_manifest(build_metadata) + ] + } + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="creates a casync release") + parser.add_argument("target_dir", type=str, help="path to a release build of openpilot to create release from") + args = parser.parse_args() + + target_dir = pathlib.Path(args.target_dir) + + with tempfile.TemporaryDirectory() as temp_dir: + casync_dir = pathlib.Path(temp_dir) / "casync" + casync_dir.mkdir(parents=True) + + manifest_file = pathlib.Path(temp_dir) / "manifest.json" + + create_casync_caibx(target_dir, casync_dir / "openpilot") + upload_casync_release(casync_dir) + manifest = create_manifest(target_dir) + + print(json.dumps(manifest, indent=2)) diff --git a/release/upload_casync_release.py b/release/upload_casync_release.py deleted file mode 100755 index 4261e298a..000000000 --- a/release/upload_casync_release.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import os -import pathlib -from openpilot.tools.lib.azure_container import AzureContainer - - -if __name__ == "__main__": - if "AZURE_TOKEN_OPENPILOT_RELEASES" in os.environ: - os.environ["AZURE_TOKEN"] = os.environ["AZURE_TOKEN_OPENPILOT_RELEASES"] - - OPENPILOT_RELEASES_CONTAINER = AzureContainer("commadist", "openpilot-releases") - - parser = argparse.ArgumentParser(description='upload casync folder to azure') - parser.add_argument("casync_dir", type=str, help="casync directory") - args = parser.parse_args() - - casync_dir = pathlib.Path(args.casync_dir) - - for f in casync_dir.rglob("*"): - if f.is_file(): - blob_name = f.relative_to(casync_dir) - print(f"uploading {f} to {blob_name}") - OPENPILOT_RELEASES_CONTAINER.upload_file(str(f), str(blob_name), overwrite=True)