consolidate casync build into fewer scripts and fix pc release build (#32225)
* less scripts
* better
* fixes
* naming
* revert
* cleanup
* lets test it
* fix that one
* and rm
* don't run this
* fix
* not here
* revert testing
* fix docs
* default here too
* t
---------
Co-authored-by: Comma Device <device@comma.ai>
old-commit-hash: 62f053bc4d
This commit is contained in:
Justin Newberry
GitHub
parent
b40ce1b085
commit
9e3e5a9437
|
|
@ -63,6 +63,7 @@ if [ -f /TICI ]; then
|
|||
rm -rf /tmp/tmp*
|
||||
rm -rf ~/.commacache
|
||||
rm -rf /dev/shm/*
|
||||
rm -rf /dev/tmp/tmp*
|
||||
|
||||
if ! systemctl is-active --quiet systemd-resolved; then
|
||||
echo "restarting resolved"
|
||||
|
|
@ -179,12 +180,14 @@ def build_git_release(String channel_name) {
|
|||
|
||||
|
||||
def build_casync_release(String channel_name, def is_release) {
|
||||
def extra_env = is_release ? "RELEASE=1" : ""
|
||||
def extra_env = is_release ? "RELEASE=1 " : ""
|
||||
def build_dir = "/data/openpilot"
|
||||
|
||||
extra_env += "TMPDIR=/data/tmp PYTHONPATH=$SOURCE_DIR"
|
||||
|
||||
return deviceStage("build casync", "tici-needs-can", [], [
|
||||
["build", "${extra_env} BUILD_DIR=/data/openpilot CASYNC_DIR=/data/casync/openpilot $SOURCE_DIR/release/create_casync_build.sh"],
|
||||
["create manifest", "$SOURCE_DIR/release/create_release_manifest.py /data/openpilot /data/manifest.json && cat /data/manifest.json"],
|
||||
["upload and cleanup", "PYTHONWARNINGS=ignore $SOURCE_DIR/release/upload_casync_release.py /data/casync && rm -rf /data/casync"],
|
||||
["build", "${extra_env} $SOURCE_DIR/release/build_release.sh ${build_dir}"],
|
||||
["package + upload", "${extra_env} $SOURCE_DIR/release/package_casync_build.py ${build_dir}"],
|
||||
])
|
||||
}
|
||||
|
||||
|
|
@ -199,8 +202,7 @@ def build_stage() {
|
|||
},
|
||||
'publish agnos': {
|
||||
pcStage("publish agnos") {
|
||||
sh "release/create_casync_agnos_release.py /tmp/casync/agnos /tmp/casync_tmp"
|
||||
sh "PYTHONWARNINGS=ignore ${env.WORKSPACE}/release/upload_casync_release.py /tmp/casync"
|
||||
sh "PYTHONWARNINGS=ignore release/package_casync_agnos.py"
|
||||
}
|
||||
}
|
||||
)
|
||||
|
|
|
|||
|
|
@ -376,11 +376,13 @@ SConscript([
|
|||
])
|
||||
if arch != "Darwin":
|
||||
SConscript([
|
||||
'system/camerad/SConscript',
|
||||
'system/sensord/SConscript',
|
||||
'system/logcatd/SConscript',
|
||||
])
|
||||
|
||||
if arch == "larch64":
|
||||
SConscript(['system/camerad/SConscript'])
|
||||
|
||||
# Build openpilot
|
||||
SConscript(['third_party/SConscript'])
|
||||
|
||||
|
|
|
|||
|
|
@ -4,8 +4,7 @@
|
|||
## terms
|
||||
|
||||
- `channel` - a named version of openpilot (git branch, casync caibx) which receives updates
|
||||
- `build` - a release which is already built for the comma 3/3x and contains only required files for running openpilot and identifying the release
|
||||
|
||||
- `build` - a copy of openpilot ready for distribution, already built for a specific device
|
||||
- `build_style` - type of build, either `debug` or `release`
|
||||
- `debug` - build with `ALLOW_DEBUG=true`, can test experimental features like longitudinal on alpha cars
|
||||
- `release` - build with `ALLOW_DEBUG=false`, experimental features disabled
|
||||
|
|
@ -22,21 +21,13 @@
|
|||
| git branches | `debug` | installed manually, experimental features enabled, build required |
|
||||
|
||||
|
||||
## creating casync build
|
||||
## build
|
||||
|
||||
`create_casync_build.sh` - creates a casync openpilot build, ready to upload to `openpilot-releases`
|
||||
`release/build_release.sh <build_dir>` - creates an openpilot build into `build_dir`, ready for distribution
|
||||
|
||||
```bash
|
||||
# run on a tici, within the directory you want to create the build from.
|
||||
# creates a prebuilt version of openpilot into BUILD_DIR and outputs the caibx
|
||||
# of a tarball containing the full prebuilt openpilot release
|
||||
BUILD_DIR=/data/openpilot_build \
|
||||
CASYNC_DIR=/data/casync \
|
||||
release/create_casync_build.sh
|
||||
```
|
||||
|
||||
`upload_casync_release.sh` - helper for uploading a casync build to `openpilot-releases`
|
||||
## packaging a casync release
|
||||
|
||||
`release/package_casync_build.py <build_dir>` - packages an openpilot build into a casync tar and uploads to `openpilot-releases`
|
||||
|
||||
## release builds
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,51 @@
|
|||
#!/usr/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
|
||||
SOURCE_DIR="$(git -C $DIR rev-parse --show-toplevel)"
|
||||
BUILD_DIR=${1:-$(mktemp -d)}
|
||||
|
||||
if [ -f /TICI ]; then
|
||||
FILES_SRC="release/files_tici"
|
||||
else
|
||||
FILES_SRC="release/files_pc"
|
||||
fi
|
||||
|
||||
echo "Building openpilot into $BUILD_DIR"
|
||||
|
||||
rm -rf $BUILD_DIR
|
||||
mkdir -p $BUILD_DIR
|
||||
|
||||
# Copy required files to BUILD_DIR
|
||||
cd $SOURCE_DIR
|
||||
cp -pR --parents $(cat release/files_common) $BUILD_DIR/
|
||||
cp -pR --parents $(cat $FILES_SRC) $BUILD_DIR/
|
||||
|
||||
# Build + cleanup
|
||||
cd $BUILD_DIR
|
||||
export PYTHONPATH="$BUILD_DIR"
|
||||
|
||||
rm -f panda/board/obj/panda.bin.signed
|
||||
rm -f panda/board/obj/panda_h7.bin.signed
|
||||
|
||||
if [ -n "$RELEASE" ]; then
|
||||
export CERT=/data/pandaextra/certs/release
|
||||
fi
|
||||
|
||||
scons -j$(nproc)
|
||||
|
||||
# Cleanup
|
||||
find . -name '*.a' -delete
|
||||
find . -name '*.o' -delete
|
||||
find . -name '*.os' -delete
|
||||
find . -name '*.pyc' -delete
|
||||
find . -name 'moc_*' -delete
|
||||
find . -name '__pycache__' -delete
|
||||
rm -rf .sconsign.dblite Jenkinsfile release/
|
||||
rm selfdrive/modeld/models/supercombo.onnx
|
||||
|
||||
# Mark as prebuilt release
|
||||
touch prebuilt
|
||||
|
||||
echo "----- openpilot has been built to $BUILD_DIR -----"
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
SOURCE_DIR=$1
|
||||
TARGET_DIR=$2
|
||||
|
||||
if [ -f /TICI ]; then
|
||||
FILES_SRC="release/files_tici"
|
||||
else
|
||||
echo "no release files set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cd $SOURCE_DIR
|
||||
cp -pR --parents $(cat release/files_common) $TARGET_DIR/
|
||||
cp -pR --parents $(cat $FILES_SRC) $TARGET_DIR/
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
#!/usr/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
|
||||
|
||||
CASYNC_DIR="${CASYNC_DIR:=/tmp/casync}"
|
||||
SOURCE_DIR="$(git -C $DIR rev-parse --show-toplevel)"
|
||||
BUILD_DIR="${BUILD_DIR:=$(mktemp -d)}"
|
||||
PYTHONPATH="$SOURCE_DIR"
|
||||
|
||||
echo "Creating casync release from $SOURCE_DIR to $CASYNC_DIR"
|
||||
|
||||
cd $SOURCE_DIR
|
||||
mkdir -p $CASYNC_DIR
|
||||
rm -rf $BUILD_DIR
|
||||
mkdir -p $BUILD_DIR
|
||||
|
||||
release/copy_build_files.sh $SOURCE_DIR $BUILD_DIR
|
||||
release/create_prebuilt.sh $BUILD_DIR
|
||||
|
||||
cd $SOURCE_DIR
|
||||
release/create_casync_release.py $BUILD_DIR $CASYNC_DIR
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
|
||||
from openpilot.system.updated.casync.common import create_casync_release, create_build_metadata_file
|
||||
from openpilot.system.version import get_build_metadata
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="creates a casync release")
|
||||
parser.add_argument("target_dir", type=str, help="target directory to build channel from")
|
||||
parser.add_argument("output_dir", type=str, help="output directory for the channel")
|
||||
args = parser.parse_args()
|
||||
|
||||
target_dir = pathlib.Path(args.target_dir)
|
||||
output_dir = pathlib.Path(args.output_dir)
|
||||
|
||||
build_metadata = get_build_metadata()
|
||||
build_metadata.openpilot.build_style = "release" if os.environ.get("RELEASE", None) is not None else "debug"
|
||||
|
||||
create_build_metadata_file(target_dir, build_metadata)
|
||||
|
||||
digest, caibx = create_casync_release(target_dir, output_dir, build_metadata.canonical)
|
||||
|
||||
print(f"Created casync release from {target_dir} to {caibx} with digest {digest}")
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
#!/usr/bin/bash -e
|
||||
|
||||
# runs on tici to create a prebuilt version of a release
|
||||
|
||||
set -ex
|
||||
|
||||
BUILD_DIR=$1
|
||||
|
||||
cd $BUILD_DIR
|
||||
|
||||
# Build
|
||||
export PYTHONPATH="$BUILD_DIR"
|
||||
|
||||
rm -f panda/board/obj/panda.bin.signed
|
||||
rm -f panda/board/obj/panda_h7.bin.signed
|
||||
|
||||
if [ -n "$RELEASE" ]; then
|
||||
export CERT=/data/pandaextra/certs/release
|
||||
fi
|
||||
|
||||
scons -j$(nproc)
|
||||
|
||||
# Cleanup
|
||||
find . -name '*.a' -delete
|
||||
find . -name '*.o' -delete
|
||||
find . -name '*.os' -delete
|
||||
find . -name '*.pyc' -delete
|
||||
find . -name 'moc_*' -delete
|
||||
find . -name '__pycache__' -delete
|
||||
rm -rf .sconsign.dblite Jenkinsfile release/
|
||||
rm selfdrive/modeld/models/supercombo.onnx
|
||||
|
||||
# Mark as prebuilt release
|
||||
touch prebuilt
|
||||
|
|
@ -1,64 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import dataclasses
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
|
||||
from openpilot.system.hardware.tici.agnos import AGNOS_MANIFEST_FILE, get_partition_path
|
||||
from openpilot.system.version import get_build_metadata
|
||||
|
||||
|
||||
BASE_URL = "https://commadist.blob.core.windows.net"
|
||||
|
||||
OPENPILOT_RELEASES = f"{BASE_URL}/openpilot-releases/openpilot"
|
||||
AGNOS_RELEASES = f"{BASE_URL}/openpilot-releases/agnos"
|
||||
|
||||
|
||||
def create_partition_manifest(partition):
|
||||
agnos_filename = os.path.basename(partition["url"]).split(".")[0]
|
||||
|
||||
return {
|
||||
"type": "partition",
|
||||
"casync": {
|
||||
"caibx": f"{AGNOS_RELEASES}/{agnos_filename}.caibx"
|
||||
},
|
||||
"path": get_partition_path(0, partition),
|
||||
"ab": True,
|
||||
"size": partition["size"],
|
||||
"full_check": partition["full_check"],
|
||||
"hash_raw": partition["hash_raw"],
|
||||
}
|
||||
|
||||
|
||||
def create_openpilot_manifest(build_metadata):
|
||||
return {
|
||||
"type": "path_tarred",
|
||||
"path": "/data/openpilot",
|
||||
"casync": {
|
||||
"caibx": f"{OPENPILOT_RELEASES}/{build_metadata.canonical}.caibx"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="creates a casync release")
|
||||
parser.add_argument("target_dir", type=str, help="directory of the channel to create manifest from")
|
||||
parser.add_argument("output_file", type=str, help="output file to put the manifest")
|
||||
args = parser.parse_args()
|
||||
|
||||
with open(pathlib.Path(args.target_dir) / AGNOS_MANIFEST_FILE) as f:
|
||||
agnos_manifest = json.load(f)
|
||||
|
||||
build_metadata = get_build_metadata(args.target_dir)
|
||||
|
||||
ret = {
|
||||
"build_metadata": dataclasses.asdict(build_metadata),
|
||||
"manifest": [
|
||||
*[create_partition_manifest(entry) for entry in agnos_manifest],
|
||||
create_openpilot_manifest(build_metadata)
|
||||
]
|
||||
}
|
||||
|
||||
with open(args.output_file, "w") as f:
|
||||
f.write(json.dumps(ret, indent=2))
|
||||
|
|
@ -8,27 +8,27 @@ import time
|
|||
from openpilot.common.basedir import BASEDIR
|
||||
from openpilot.system.hardware.tici.agnos import StreamingDecompressor, unsparsify, noop, AGNOS_MANIFEST_FILE
|
||||
from openpilot.system.updated.casync.common import create_casync_from_file
|
||||
from release.package_casync_build import upload_casync_release
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="creates a casync release")
|
||||
parser.add_argument("output_dir", type=str, help="output directory for the channel")
|
||||
parser.add_argument("working_dir", type=str, help="working directory")
|
||||
parser.add_argument("--manifest", type=str, help="json manifest to create agnos release from", \
|
||||
default=str(pathlib.Path(BASEDIR) / AGNOS_MANIFEST_FILE))
|
||||
args = parser.parse_args()
|
||||
|
||||
output_dir = pathlib.Path(args.output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
working_dir = pathlib.Path(args.working_dir)
|
||||
working_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
manifest_file = pathlib.Path(args.manifest)
|
||||
|
||||
with tempfile.NamedTemporaryFile(dir=str(working_dir)) as entry_file:
|
||||
entry_path = pathlib.Path(entry_file.name)
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
working_dir = pathlib.Path(temp_dir)
|
||||
casync_dir = working_dir / "casync"
|
||||
casync_dir.mkdir()
|
||||
|
||||
agnos_casync_dir = casync_dir / "agnos"
|
||||
agnos_casync_dir.mkdir()
|
||||
|
||||
entry_path = working_dir / "entry"
|
||||
|
||||
with open(manifest_file) as f:
|
||||
manifest = json.load(f)
|
||||
|
|
@ -53,5 +53,7 @@ if __name__ == "__main__":
|
|||
|
||||
start = time.monotonic()
|
||||
agnos_filename = os.path.basename(entry["url"]).split(".")[0]
|
||||
create_casync_from_file(entry_path, output_dir, agnos_filename)
|
||||
create_casync_from_file(entry_path, agnos_casync_dir, agnos_filename)
|
||||
print(f"created casnc in {time.monotonic() - start}")
|
||||
|
||||
upload_casync_release(casync_dir)
|
||||
|
|
@ -0,0 +1,108 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# packages a casync release, uploads to azure, and creates a manifest
|
||||
|
||||
import argparse
|
||||
import dataclasses
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import tempfile
|
||||
|
||||
from openpilot.system.hardware.tici.agnos import AGNOS_MANIFEST_FILE, get_partition_path
|
||||
from openpilot.system.updated.casync.common import create_build_metadata_file, create_casync_release
|
||||
from openpilot.system.version import get_build_metadata
|
||||
from openpilot.tools.lib.azure_container import AzureContainer
|
||||
|
||||
|
||||
BASE_URL = "https://commadist.blob.core.windows.net"
|
||||
|
||||
OPENPILOT_RELEASES = f"{BASE_URL}/openpilot-releases/openpilot"
|
||||
AGNOS_RELEASES = f"{BASE_URL}/openpilot-releases/agnos"
|
||||
|
||||
|
||||
def create_casync_caibx(target_dir: pathlib.Path, output_dir: pathlib.Path):
|
||||
output_dir.mkdir()
|
||||
build_metadata = get_build_metadata()
|
||||
build_metadata.openpilot.build_style = "release" if os.environ.get("RELEASE", None) is not None else "debug"
|
||||
|
||||
create_build_metadata_file(target_dir, build_metadata)
|
||||
|
||||
digest, caibx = create_casync_release(target_dir, output_dir, build_metadata.canonical)
|
||||
|
||||
print(f"Created casync release from {target_dir} to {caibx} with digest {digest}")
|
||||
|
||||
|
||||
def upload_casync_release(casync_dir: pathlib.Path):
|
||||
if "AZURE_TOKEN_OPENPILOT_RELEASES" in os.environ:
|
||||
os.environ["AZURE_TOKEN"] = os.environ["AZURE_TOKEN_OPENPILOT_RELEASES"]
|
||||
|
||||
OPENPILOT_RELEASES_CONTAINER = AzureContainer("commadist", "openpilot-releases")
|
||||
|
||||
for f in casync_dir.rglob("*"):
|
||||
if f.is_file():
|
||||
blob_name = f.relative_to(casync_dir)
|
||||
print(f"uploading {f} to {blob_name}")
|
||||
OPENPILOT_RELEASES_CONTAINER.upload_file(str(f), str(blob_name), overwrite=True)
|
||||
|
||||
|
||||
def create_partition_manifest(partition):
|
||||
agnos_filename = os.path.basename(partition["url"]).split(".")[0]
|
||||
|
||||
return {
|
||||
"type": "partition",
|
||||
"casync": {
|
||||
"caibx": f"{AGNOS_RELEASES}/{agnos_filename}.caibx"
|
||||
},
|
||||
"path": get_partition_path(0, partition),
|
||||
"ab": True,
|
||||
"size": partition["size"],
|
||||
"full_check": partition["full_check"],
|
||||
"hash_raw": partition["hash_raw"],
|
||||
}
|
||||
|
||||
|
||||
def create_openpilot_manifest(build_metadata):
|
||||
return {
|
||||
"type": "path_tarred",
|
||||
"path": "/data/openpilot",
|
||||
"casync": {
|
||||
"caibx": f"{OPENPILOT_RELEASES}/{build_metadata.canonical}.caibx"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def create_manifest(target_dir):
|
||||
with open(pathlib.Path(target_dir) / AGNOS_MANIFEST_FILE) as f:
|
||||
agnos_manifest = json.load(f)
|
||||
|
||||
build_metadata = get_build_metadata(args.target_dir)
|
||||
|
||||
return {
|
||||
"build_metadata": dataclasses.asdict(build_metadata),
|
||||
"manifest": [
|
||||
*[create_partition_manifest(entry) for entry in agnos_manifest],
|
||||
create_openpilot_manifest(build_metadata)
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="creates a casync release")
|
||||
parser.add_argument("target_dir", type=str, help="path to a release build of openpilot to create release from")
|
||||
args = parser.parse_args()
|
||||
|
||||
target_dir = pathlib.Path(args.target_dir)
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
casync_dir = pathlib.Path(temp_dir) / "casync"
|
||||
casync_dir.mkdir(parents=True)
|
||||
|
||||
manifest_file = pathlib.Path(temp_dir) / "manifest.json"
|
||||
|
||||
create_casync_caibx(target_dir, casync_dir / "openpilot")
|
||||
upload_casync_release(casync_dir)
|
||||
manifest = create_manifest(target_dir)
|
||||
|
||||
print(json.dumps(manifest, indent=2))
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
from openpilot.tools.lib.azure_container import AzureContainer
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if "AZURE_TOKEN_OPENPILOT_RELEASES" in os.environ:
|
||||
os.environ["AZURE_TOKEN"] = os.environ["AZURE_TOKEN_OPENPILOT_RELEASES"]
|
||||
|
||||
OPENPILOT_RELEASES_CONTAINER = AzureContainer("commadist", "openpilot-releases")
|
||||
|
||||
parser = argparse.ArgumentParser(description='upload casync folder to azure')
|
||||
parser.add_argument("casync_dir", type=str, help="casync directory")
|
||||
args = parser.parse_args()
|
||||
|
||||
casync_dir = pathlib.Path(args.casync_dir)
|
||||
|
||||
for f in casync_dir.rglob("*"):
|
||||
if f.is_file():
|
||||
blob_name = f.relative_to(casync_dir)
|
||||
print(f"uploading {f} to {blob_name}")
|
||||
OPENPILOT_RELEASES_CONTAINER.upload_file(str(f), str(blob_name), overwrite=True)
|
||||
Loading…
Reference in New Issue