mirror of https://github.com/1okko/openpilot.git
revert changes to release/ (#32364)
* revert changes to release/ * put those back * revert Jenkinsfile
This commit is contained in:
Adeeb Shihadeh
GitHub
parent
1de6428896
commit
1c42b8a05f
|
|
@ -9,11 +9,6 @@ def retryWithDelay(int maxRetries, int delay, Closure body) {
|
|||
throw Exception("Failed after ${maxRetries} retries")
|
||||
}
|
||||
|
||||
// check if started by timer: https://gist.github.com/aaclarker/75b8a0eb2b4d600779f84f8e849f2c37
|
||||
def isJobStartedByTimer() {
|
||||
return currentBuild.getBuildCauses()[0]["shortDescription"].matches("Started by timer");
|
||||
}
|
||||
|
||||
def device(String ip, String step_label, String cmd) {
|
||||
withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) {
|
||||
def ssh_cmd = """
|
||||
|
|
@ -31,7 +26,6 @@ export SOURCE_DIR=${env.SOURCE_DIR}
|
|||
export GIT_BRANCH=${env.GIT_BRANCH}
|
||||
export GIT_COMMIT=${env.GIT_COMMIT}
|
||||
export AZURE_TOKEN='${env.AZURE_TOKEN}'
|
||||
export AZURE_TOKEN_OPENPILOT_RELEASES='${env.AZURE_TOKEN_OPENPILOT_RELEASES}'
|
||||
export MAPBOX_TOKEN='${env.MAPBOX_TOKEN}'
|
||||
# only use 1 thread for tici tests since most require HIL
|
||||
export PYTEST_ADDOPTS="-n 0"
|
||||
|
|
@ -111,7 +105,7 @@ def pcStage(String stageName, Closure body) {
|
|||
|
||||
checkout scm
|
||||
|
||||
def dockerArgs = "--user=batman -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/scons_cache:/tmp/scons_cache -e PYTHONPATH=${env.WORKSPACE} -e AZURE_TOKEN_OPENPILOT_RELEASES='${env.AZURE_TOKEN_OPENPILOT_RELEASES}' --cpus=8 --memory 16g -e PYTEST_ADDOPTS='-n8'";
|
||||
def dockerArgs = "--user=batman -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/scons_cache:/tmp/scons_cache -e PYTHONPATH=${env.WORKSPACE} --cpus=8 --memory 16g -e PYTEST_ADDOPTS='-n8'";
|
||||
|
||||
def openpilot_base = retryWithDelay (3, 15) {
|
||||
return docker.build("openpilot-base:build-${env.GIT_COMMIT}", "-f Dockerfile.openpilot_base .")
|
||||
|
|
@ -119,7 +113,7 @@ def pcStage(String stageName, Closure body) {
|
|||
|
||||
lock(resource: "", label: 'pc', inversePrecedence: true, quantity: 1) {
|
||||
openpilot_base.inside(dockerArgs) {
|
||||
timeout(time: 25, unit: 'MINUTES') {
|
||||
timeout(time: 20, unit: 'MINUTES') {
|
||||
try {
|
||||
retryWithDelay (3, 15) {
|
||||
sh "git config --global --add safe.directory '*'"
|
||||
|
|
@ -141,56 +135,14 @@ def pcStage(String stageName, Closure body) {
|
|||
def setupCredentials() {
|
||||
withCredentials([
|
||||
string(credentialsId: 'azure_token', variable: 'AZURE_TOKEN'),
|
||||
string(credentialsId: 'azure_token_openpilot_releases', variable: 'AZURE_TOKEN_OPENPILOT_RELEASES'),
|
||||
string(credentialsId: 'mapbox_token', variable: 'MAPBOX_TOKEN')
|
||||
]) {
|
||||
env.AZURE_TOKEN = "${AZURE_TOKEN}"
|
||||
env.AZURE_TOKEN_OPENPILOT_RELEASES = "${AZURE_TOKEN_OPENPILOT_RELEASES}"
|
||||
env.MAPBOX_TOKEN = "${MAPBOX_TOKEN}"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def build_git_release(String channel_name) {
|
||||
return parallel (
|
||||
"${channel_name} (git)": {
|
||||
deviceStage("build git", "tici-needs-can", [], [
|
||||
["build ${channel_name}", "RELEASE_BRANCH=${channel_name} $SOURCE_DIR/release/build_git_release.sh"],
|
||||
])
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def build_casync_release(String channel_name, def is_release) {
|
||||
def extra_env = is_release ? "RELEASE=1 " : ""
|
||||
def build_dir = "/data/openpilot"
|
||||
|
||||
extra_env += "TMPDIR=/data/tmp PYTHONPATH=$SOURCE_DIR"
|
||||
|
||||
return deviceStage("build casync", "tici-needs-can", [], [
|
||||
["build", "${extra_env} $SOURCE_DIR/release/build_release.sh ${build_dir}"],
|
||||
["package + upload", "${extra_env} $SOURCE_DIR/release/package_casync_build.py ${build_dir}"],
|
||||
])
|
||||
}
|
||||
|
||||
|
||||
def build_stage() {
|
||||
return parallel (
|
||||
'nightly': {
|
||||
build_casync_release("nightly", true);
|
||||
},
|
||||
'master': {
|
||||
build_casync_release("master", false);
|
||||
},
|
||||
'publish agnos': {
|
||||
pcStage("publish agnos") {
|
||||
sh "PYTHONWARNINGS=ignore release/package_casync_agnos.py"
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
node {
|
||||
env.CI = "1"
|
||||
env.PYTHONWARNINGS = "error"
|
||||
|
|
@ -205,105 +157,94 @@ node {
|
|||
'testing-closet*', 'hotfix-*']
|
||||
def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*')
|
||||
|
||||
def nightlyBranch = "master"
|
||||
|
||||
def props = [];
|
||||
|
||||
if (env.BRANCH_NAME == nightlyBranch) {
|
||||
props.add(pipelineTriggers([
|
||||
cron('0 9 * * *') // at 2am PST (9am UTC) every night
|
||||
]))
|
||||
if (env.BRANCH_NAME != 'master') {
|
||||
properties([
|
||||
disableConcurrentBuilds(abortPrevious: true)
|
||||
])
|
||||
}
|
||||
|
||||
if (env.BRANCH_NAME != "master") {
|
||||
props.add(disableConcurrentBuilds(abortPrevious: true))
|
||||
}
|
||||
|
||||
properties(props);
|
||||
|
||||
try {
|
||||
if (env.BRANCH_NAME == 'devel-staging') {
|
||||
build_git_release("release3-staging")
|
||||
deviceStage("build release3-staging", "tici-needs-can", [], [
|
||||
["build release3-staging", "RELEASE_BRANCH=release3-staging $SOURCE_DIR/release/build_release.sh"],
|
||||
])
|
||||
}
|
||||
|
||||
if (env.BRANCH_NAME == 'master-ci') {
|
||||
build_git_release("nightly")
|
||||
deviceStage("build nightly", "tici-needs-can", [], [
|
||||
["build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"],
|
||||
])
|
||||
}
|
||||
|
||||
if (!env.BRANCH_NAME.matches(excludeRegex)) {
|
||||
parallel (
|
||||
// tici tests
|
||||
'onroad tests': {
|
||||
deviceStage("onroad", "tici-needs-can", [], [
|
||||
// TODO: ideally, this test runs in master-ci, but it takes 5+m to build it
|
||||
//["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR $SOURCE_DIR/scripts/retry.sh ./build_devel.sh"],
|
||||
["build openpilot", "cd selfdrive/manager && ./build.py"],
|
||||
["check dirty", "release/check-dirty.sh"],
|
||||
["onroad tests", "pytest selfdrive/test/test_onroad.py -s"],
|
||||
["time to onroad", "pytest selfdrive/test/test_time_to_onroad.py"],
|
||||
])
|
||||
},
|
||||
'HW + Unit Tests': {
|
||||
deviceStage("tici-hardware", "tici-common", ["UNSAFE=1"], [
|
||||
["build", "cd selfdrive/manager && ./build.py"],
|
||||
["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py"],
|
||||
["test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"],
|
||||
["test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py"],
|
||||
["test pigeond", "pytest system/ubloxd/tests/test_pigeond.py"],
|
||||
["test manager", "pytest selfdrive/manager/test/test_manager.py"],
|
||||
])
|
||||
},
|
||||
'loopback': {
|
||||
deviceStage("loopback", "tici-loopback", ["UNSAFE=1"], [
|
||||
["build openpilot", "cd selfdrive/manager && ./build.py"],
|
||||
["test boardd loopback", "pytest selfdrive/boardd/tests/test_boardd_loopback.py"],
|
||||
])
|
||||
},
|
||||
'camerad': {
|
||||
deviceStage("AR0231", "tici-ar0231", ["UNSAFE=1"], [
|
||||
["build", "cd selfdrive/manager && ./build.py"],
|
||||
["test camerad", "pytest system/camerad/test/test_camerad.py"],
|
||||
["test exposure", "pytest system/camerad/test/test_exposure.py"],
|
||||
])
|
||||
deviceStage("OX03C10", "tici-ox03c10", ["UNSAFE=1"], [
|
||||
["build", "cd selfdrive/manager && ./build.py"],
|
||||
["test camerad", "pytest system/camerad/test/test_camerad.py"],
|
||||
["test exposure", "pytest system/camerad/test/test_exposure.py"],
|
||||
])
|
||||
},
|
||||
'sensord': {
|
||||
deviceStage("LSM + MMC", "tici-lsmc", ["UNSAFE=1"], [
|
||||
["build", "cd selfdrive/manager && ./build.py"],
|
||||
["test sensord", "pytest system/sensord/tests/test_sensord.py"],
|
||||
])
|
||||
deviceStage("BMX + LSM", "tici-bmx-lsm", ["UNSAFE=1"], [
|
||||
["build", "cd selfdrive/manager && ./build.py"],
|
||||
["test sensord", "pytest system/sensord/tests/test_sensord.py"],
|
||||
])
|
||||
},
|
||||
'replay': {
|
||||
deviceStage("model-replay", "tici-replay", ["UNSAFE=1"], [
|
||||
["build", "cd selfdrive/manager && ./build.py"],
|
||||
["model replay", "selfdrive/test/process_replay/model_replay.py"],
|
||||
])
|
||||
},
|
||||
'tizi': {
|
||||
deviceStage("tizi", "tizi", ["UNSAFE=1"], [
|
||||
["build openpilot", "cd selfdrive/manager && ./build.py"],
|
||||
["test boardd loopback", "SINGLE_PANDA=1 pytest selfdrive/boardd/tests/test_boardd_loopback.py"],
|
||||
["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py"],
|
||||
["test amp", "pytest system/hardware/tici/tests/test_amplifier.py"],
|
||||
["test hw", "pytest system/hardware/tici/tests/test_hardware.py"],
|
||||
["test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py"],
|
||||
])
|
||||
},
|
||||
)
|
||||
}
|
||||
parallel (
|
||||
// tici tests
|
||||
'onroad tests': {
|
||||
deviceStage("onroad", "tici-needs-can", [], [
|
||||
// TODO: ideally, this test runs in master-ci, but it takes 5+m to build it
|
||||
//["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR $SOURCE_DIR/scripts/retry.sh ./build_devel.sh"],
|
||||
["build openpilot", "cd selfdrive/manager && ./build.py"],
|
||||
["check dirty", "release/check-dirty.sh"],
|
||||
["onroad tests", "pytest selfdrive/test/test_onroad.py -s"],
|
||||
["time to onroad", "pytest selfdrive/test/test_time_to_onroad.py"],
|
||||
])
|
||||
},
|
||||
'HW + Unit Tests': {
|
||||
deviceStage("tici-hardware", "tici-common", ["UNSAFE=1"], [
|
||||
["build", "cd selfdrive/manager && ./build.py"],
|
||||
["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py"],
|
||||
["test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"],
|
||||
["test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py"],
|
||||
["test pigeond", "pytest system/sensord/tests/test_pigeond.py"],
|
||||
["test manager", "pytest selfdrive/manager/test/test_manager.py"],
|
||||
])
|
||||
},
|
||||
'loopback': {
|
||||
deviceStage("loopback", "tici-loopback", ["UNSAFE=1"], [
|
||||
["build openpilot", "cd selfdrive/manager && ./build.py"],
|
||||
["test boardd loopback", "pytest selfdrive/boardd/tests/test_boardd_loopback.py"],
|
||||
])
|
||||
},
|
||||
'camerad': {
|
||||
deviceStage("AR0231", "tici-ar0231", ["UNSAFE=1"], [
|
||||
["build", "cd selfdrive/manager && ./build.py"],
|
||||
["test camerad", "pytest system/camerad/test/test_camerad.py"],
|
||||
["test exposure", "pytest system/camerad/test/test_exposure.py"],
|
||||
])
|
||||
deviceStage("OX03C10", "tici-ox03c10", ["UNSAFE=1"], [
|
||||
["build", "cd selfdrive/manager && ./build.py"],
|
||||
["test camerad", "pytest system/camerad/test/test_camerad.py"],
|
||||
["test exposure", "pytest system/camerad/test/test_exposure.py"],
|
||||
])
|
||||
},
|
||||
'sensord': {
|
||||
deviceStage("LSM + MMC", "tici-lsmc", ["UNSAFE=1"], [
|
||||
["build", "cd selfdrive/manager && ./build.py"],
|
||||
["test sensord", "pytest system/sensord/tests/test_sensord.py"],
|
||||
])
|
||||
deviceStage("BMX + LSM", "tici-bmx-lsm", ["UNSAFE=1"], [
|
||||
["build", "cd selfdrive/manager && ./build.py"],
|
||||
["test sensord", "pytest system/sensord/tests/test_sensord.py"],
|
||||
])
|
||||
},
|
||||
'replay': {
|
||||
deviceStage("model-replay", "tici-replay", ["UNSAFE=1"], [
|
||||
["build", "cd selfdrive/manager && ./build.py"],
|
||||
["model replay", "selfdrive/test/process_replay/model_replay.py"],
|
||||
])
|
||||
},
|
||||
'tizi': {
|
||||
deviceStage("tizi", "tizi", ["UNSAFE=1"], [
|
||||
["build openpilot", "cd selfdrive/manager && ./build.py"],
|
||||
["test boardd loopback", "SINGLE_PANDA=1 pytest selfdrive/boardd/tests/test_boardd_loopback.py"],
|
||||
["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py"],
|
||||
["test amp", "pytest system/hardware/tici/tests/test_amplifier.py"],
|
||||
["test hw", "pytest system/hardware/tici/tests/test_hardware.py"],
|
||||
["test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py"],
|
||||
])
|
||||
},
|
||||
|
||||
if (env.BRANCH_NAME == nightlyBranch && isJobStartedByTimer()) {
|
||||
stage('build release') {
|
||||
build_stage()
|
||||
}
|
||||
)
|
||||
}
|
||||
} catch (Exception e) {
|
||||
currentBuild.result = 'FAILED'
|
||||
|
|
|
|||
|
|
@ -1,34 +0,0 @@
|
|||
# openpilot releases
|
||||
|
||||
|
||||
## terms
|
||||
|
||||
- `channel` - a named version of openpilot (git branch, casync caibx) which receives updates
|
||||
- `build` - a copy of openpilot ready for distribution, already built for a specific device
|
||||
- `build_style` - type of build, either `debug` or `release`
|
||||
- `debug` - build with `ALLOW_DEBUG=true`, can test experimental features like longitudinal on alpha cars
|
||||
- `release` - build with `ALLOW_DEBUG=false`, experimental features disabled
|
||||
|
||||
|
||||
## openpilot channels
|
||||
|
||||
| channel | build_style | description |
|
||||
| ----------- | ----------- | ---------- |
|
||||
| release | `release` | stable release of openpilot |
|
||||
| staging | `release` | release candidate of openpilot for final verification |
|
||||
| nightly | `release` | generated nightly from last commit passing CI tests |
|
||||
| master | `debug` | current master commit with experimental features enabled |
|
||||
| git branches | `debug` | installed manually, experimental features enabled, build required |
|
||||
|
||||
|
||||
## build
|
||||
|
||||
`release/build_release.sh <build_dir>` - creates an openpilot build into `build_dir`, ready for distribution
|
||||
|
||||
## packaging a casync release
|
||||
|
||||
`release/package_casync_build.py <build_dir>` - packages an openpilot build into a casync tar and uploads to `openpilot-releases`
|
||||
|
||||
## release builds
|
||||
|
||||
to create a release build, set `RELEASE=1` environment variable when running the build script
|
||||
|
|
@ -1,105 +0,0 @@
|
|||
#!/usr/bin/bash -e
|
||||
|
||||
# git diff --name-status origin/release3-staging | grep "^A" | less
|
||||
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
|
||||
|
||||
cd $DIR
|
||||
|
||||
BUILD_DIR=/data/openpilot
|
||||
SOURCE_DIR="$(git rev-parse --show-toplevel)"
|
||||
|
||||
if [ -f /TICI ]; then
|
||||
FILES_SRC="release/files_tici"
|
||||
else
|
||||
echo "no release files set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$RELEASE_BRANCH" ]; then
|
||||
echo "RELEASE_BRANCH is not set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
# set git identity
|
||||
source $DIR/identity.sh
|
||||
|
||||
echo "[-] Setting up repo T=$SECONDS"
|
||||
rm -rf $BUILD_DIR
|
||||
mkdir -p $BUILD_DIR
|
||||
cd $BUILD_DIR
|
||||
git init
|
||||
git remote add origin git@github.com:commaai/openpilot.git
|
||||
git checkout --orphan $RELEASE_BRANCH
|
||||
|
||||
# do the files copy
|
||||
echo "[-] copying files T=$SECONDS"
|
||||
cd $SOURCE_DIR
|
||||
cp -pR --parents $(cat release/files_common) $BUILD_DIR/
|
||||
cp -pR --parents $(cat $FILES_SRC) $BUILD_DIR/
|
||||
|
||||
# in the directory
|
||||
cd $BUILD_DIR
|
||||
|
||||
rm -f panda/board/obj/panda.bin.signed
|
||||
rm -f panda/board/obj/panda_h7.bin.signed
|
||||
|
||||
VERSION=$(cat common/version.h | awk -F[\"-] '{print $2}')
|
||||
echo "#define COMMA_VERSION \"$VERSION-release\"" > common/version.h
|
||||
|
||||
echo "[-] committing version $VERSION T=$SECONDS"
|
||||
git add -f .
|
||||
git commit -a -m "openpilot v$VERSION release"
|
||||
|
||||
# Build
|
||||
export PYTHONPATH="$BUILD_DIR"
|
||||
scons -j$(nproc)
|
||||
|
||||
# release panda fw
|
||||
CERT=/data/pandaextra/certs/release RELEASE=1 scons -j$(nproc) panda/
|
||||
|
||||
# Ensure no submodules in release
|
||||
if test "$(git submodule--helper list | wc -l)" -gt "0"; then
|
||||
echo "submodules found:"
|
||||
git submodule--helper list
|
||||
exit 1
|
||||
fi
|
||||
git submodule status
|
||||
|
||||
# Cleanup
|
||||
find . -name '*.a' -delete
|
||||
find . -name '*.o' -delete
|
||||
find . -name '*.os' -delete
|
||||
find . -name '*.pyc' -delete
|
||||
find . -name 'moc_*' -delete
|
||||
find . -name '__pycache__' -delete
|
||||
rm -rf .sconsign.dblite Jenkinsfile release/
|
||||
rm selfdrive/modeld/models/supercombo.onnx
|
||||
|
||||
# Restore third_party
|
||||
git checkout third_party/
|
||||
|
||||
# Mark as prebuilt release
|
||||
touch prebuilt
|
||||
|
||||
# Add built files to git
|
||||
git add -f .
|
||||
git commit --amend -m "openpilot v$VERSION"
|
||||
|
||||
# Run tests
|
||||
TEST_FILES="tools/"
|
||||
cd $SOURCE_DIR
|
||||
cp -pR -n --parents $TEST_FILES $BUILD_DIR/
|
||||
cd $BUILD_DIR
|
||||
RELEASE=1 selfdrive/test/test_onroad.py
|
||||
#selfdrive/manager/test/test_manager.py
|
||||
#selfdrive/car/tests/test_car_interfaces.py
|
||||
rm -rf $TEST_FILES
|
||||
|
||||
if [ ! -z "$RELEASE_BRANCH" ]; then
|
||||
echo "[-] pushing release T=$SECONDS"
|
||||
git push -f origin $RELEASE_BRANCH:$RELEASE_BRANCH
|
||||
fi
|
||||
|
||||
echo "[-] done T=$SECONDS"
|
||||
|
|
@ -1,40 +1,72 @@
|
|||
#!/usr/bin/bash
|
||||
#!/usr/bin/bash -e
|
||||
|
||||
set -e
|
||||
# git diff --name-status origin/release3-staging | grep "^A" | less
|
||||
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
|
||||
SOURCE_DIR="$(git -C $DIR rev-parse --show-toplevel)"
|
||||
BUILD_DIR=${1:-$(mktemp -d)}
|
||||
|
||||
cd $DIR
|
||||
|
||||
BUILD_DIR=/data/openpilot
|
||||
SOURCE_DIR="$(git rev-parse --show-toplevel)"
|
||||
|
||||
if [ -f /TICI ]; then
|
||||
FILES_SRC="release/files_tici"
|
||||
else
|
||||
FILES_SRC="release/files_pc"
|
||||
echo "no release files set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Building openpilot into $BUILD_DIR"
|
||||
if [ -z "$RELEASE_BRANCH" ]; then
|
||||
echo "RELEASE_BRANCH is not set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
||||
# set git identity
|
||||
source $DIR/identity.sh
|
||||
|
||||
echo "[-] Setting up repo T=$SECONDS"
|
||||
rm -rf $BUILD_DIR
|
||||
mkdir -p $BUILD_DIR
|
||||
cd $BUILD_DIR
|
||||
git init
|
||||
git remote add origin git@github.com:commaai/openpilot.git
|
||||
git checkout --orphan $RELEASE_BRANCH
|
||||
|
||||
# Copy required files to BUILD_DIR
|
||||
# do the files copy
|
||||
echo "[-] copying files T=$SECONDS"
|
||||
cd $SOURCE_DIR
|
||||
cp -pR --parents $(cat release/files_common) $BUILD_DIR/
|
||||
cp -pR --parents $(cat $FILES_SRC) $BUILD_DIR/
|
||||
|
||||
# Build + cleanup
|
||||
# in the directory
|
||||
cd $BUILD_DIR
|
||||
export PYTHONPATH="$BUILD_DIR"
|
||||
|
||||
rm -f panda/board/obj/panda.bin.signed
|
||||
rm -f panda/board/obj/panda_h7.bin.signed
|
||||
|
||||
if [ -n "$RELEASE" ]; then
|
||||
export CERT=/data/pandaextra/certs/release
|
||||
fi
|
||||
VERSION=$(cat common/version.h | awk -F[\"-] '{print $2}')
|
||||
echo "#define COMMA_VERSION \"$VERSION-release\"" > common/version.h
|
||||
|
||||
echo "[-] committing version $VERSION T=$SECONDS"
|
||||
git add -f .
|
||||
git commit -a -m "openpilot v$VERSION release"
|
||||
|
||||
# Build
|
||||
export PYTHONPATH="$BUILD_DIR"
|
||||
scons -j$(nproc)
|
||||
|
||||
# release panda fw
|
||||
CERT=/data/pandaextra/certs/release RELEASE=1 scons -j$(nproc) panda/
|
||||
|
||||
# Ensure no submodules in release
|
||||
if test "$(git submodule--helper list | wc -l)" -gt "0"; then
|
||||
echo "submodules found:"
|
||||
git submodule--helper list
|
||||
exit 1
|
||||
fi
|
||||
git submodule status
|
||||
|
||||
# Cleanup
|
||||
find . -name '*.a' -delete
|
||||
find . -name '*.o' -delete
|
||||
|
|
@ -45,7 +77,29 @@ find . -name '__pycache__' -delete
|
|||
rm -rf .sconsign.dblite Jenkinsfile release/
|
||||
rm selfdrive/modeld/models/supercombo.onnx
|
||||
|
||||
# Restore third_party
|
||||
git checkout third_party/
|
||||
|
||||
# Mark as prebuilt release
|
||||
touch prebuilt
|
||||
|
||||
echo "----- openpilot has been built to $BUILD_DIR -----"
|
||||
# Add built files to git
|
||||
git add -f .
|
||||
git commit --amend -m "openpilot v$VERSION"
|
||||
|
||||
# Run tests
|
||||
TEST_FILES="tools/"
|
||||
cd $SOURCE_DIR
|
||||
cp -pR -n --parents $TEST_FILES $BUILD_DIR/
|
||||
cd $BUILD_DIR
|
||||
RELEASE=1 selfdrive/test/test_onroad.py
|
||||
#selfdrive/manager/test/test_manager.py
|
||||
selfdrive/car/tests/test_car_interfaces.py
|
||||
rm -rf $TEST_FILES
|
||||
|
||||
if [ ! -z "$RELEASE_BRANCH" ]; then
|
||||
echo "[-] pushing release T=$SECONDS"
|
||||
git push -f origin $RELEASE_BRANCH:$RELEASE_BRANCH
|
||||
fi
|
||||
|
||||
echo "[-] done T=$SECONDS"
|
||||
|
|
|
|||
|
|
@ -1,59 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import tempfile
|
||||
import time
|
||||
from openpilot.common.basedir import BASEDIR
|
||||
from openpilot.system.hardware.tici.agnos import StreamingDecompressor, unsparsify, noop, AGNOS_MANIFEST_FILE
|
||||
from openpilot.system.updated.casync.common import create_casync_from_file
|
||||
from release.package_casync_build import upload_casync_release
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="creates a casync release")
|
||||
parser.add_argument("--manifest", type=str, help="json manifest to create agnos release from", \
|
||||
default=str(pathlib.Path(BASEDIR) / AGNOS_MANIFEST_FILE))
|
||||
args = parser.parse_args()
|
||||
|
||||
manifest_file = pathlib.Path(args.manifest)
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
working_dir = pathlib.Path(temp_dir)
|
||||
casync_dir = working_dir / "casync"
|
||||
casync_dir.mkdir()
|
||||
|
||||
agnos_casync_dir = casync_dir / "agnos"
|
||||
agnos_casync_dir.mkdir()
|
||||
|
||||
entry_path = working_dir / "entry"
|
||||
|
||||
with open(manifest_file) as f:
|
||||
manifest = json.load(f)
|
||||
|
||||
for entry in manifest:
|
||||
print(f"creating casync agnos build from {entry}")
|
||||
start = time.monotonic()
|
||||
downloader = StreamingDecompressor(entry['url'])
|
||||
|
||||
parse_func = unsparsify if entry['sparse'] else noop
|
||||
|
||||
parsed_chunks = parse_func(downloader)
|
||||
|
||||
size = entry["size"]
|
||||
|
||||
cur = 0
|
||||
with open(entry_path, "wb") as f:
|
||||
for chunk in parsed_chunks:
|
||||
f.write(chunk)
|
||||
|
||||
print(f"downloaded in {time.monotonic() - start}")
|
||||
|
||||
start = time.monotonic()
|
||||
agnos_filename = os.path.basename(entry["url"]).split(".")[0]
|
||||
create_casync_from_file(entry_path, agnos_casync_dir, agnos_filename)
|
||||
print(f"created casnc in {time.monotonic() - start}")
|
||||
|
||||
upload_casync_release(casync_dir)
|
||||
|
|
@ -1,108 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# packages a casync release, uploads to azure, and creates a manifest
|
||||
|
||||
import argparse
|
||||
import dataclasses
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import tempfile
|
||||
|
||||
from openpilot.system.hardware.tici.agnos import AGNOS_MANIFEST_FILE, get_partition_path
|
||||
from openpilot.system.updated.casync.common import create_build_metadata_file, create_casync_release
|
||||
from openpilot.system.version import get_build_metadata
|
||||
from openpilot.tools.lib.azure_container import AzureContainer
|
||||
|
||||
|
||||
BASE_URL = "https://commadist.blob.core.windows.net"
|
||||
|
||||
OPENPILOT_RELEASES = f"{BASE_URL}/openpilot-releases/openpilot"
|
||||
AGNOS_RELEASES = f"{BASE_URL}/openpilot-releases/agnos"
|
||||
|
||||
|
||||
def create_casync_caibx(target_dir: pathlib.Path, output_dir: pathlib.Path):
|
||||
output_dir.mkdir()
|
||||
build_metadata = get_build_metadata()
|
||||
build_metadata.openpilot.build_style = "release" if os.environ.get("RELEASE", None) is not None else "debug"
|
||||
|
||||
create_build_metadata_file(target_dir, build_metadata)
|
||||
|
||||
digest, caibx = create_casync_release(target_dir, output_dir, build_metadata.canonical)
|
||||
|
||||
print(f"Created casync release from {target_dir} to {caibx} with digest {digest}")
|
||||
|
||||
|
||||
def upload_casync_release(casync_dir: pathlib.Path):
|
||||
if "AZURE_TOKEN_OPENPILOT_RELEASES" in os.environ:
|
||||
os.environ["AZURE_TOKEN"] = os.environ["AZURE_TOKEN_OPENPILOT_RELEASES"]
|
||||
|
||||
OPENPILOT_RELEASES_CONTAINER = AzureContainer("commadist", "openpilot-releases")
|
||||
|
||||
for f in casync_dir.rglob("*"):
|
||||
if f.is_file():
|
||||
blob_name = f.relative_to(casync_dir)
|
||||
print(f"uploading {f} to {blob_name}")
|
||||
OPENPILOT_RELEASES_CONTAINER.upload_file(str(f), str(blob_name), overwrite=True)
|
||||
|
||||
|
||||
def create_partition_manifest(partition):
|
||||
agnos_filename = os.path.basename(partition["url"]).split(".")[0]
|
||||
|
||||
return {
|
||||
"type": "partition",
|
||||
"casync": {
|
||||
"caibx": f"{AGNOS_RELEASES}/{agnos_filename}.caibx"
|
||||
},
|
||||
"path": get_partition_path(0, partition),
|
||||
"ab": True,
|
||||
"size": partition["size"],
|
||||
"full_check": partition["full_check"],
|
||||
"hash_raw": partition["hash_raw"],
|
||||
}
|
||||
|
||||
|
||||
def create_openpilot_manifest(build_metadata):
|
||||
return {
|
||||
"type": "path_tarred",
|
||||
"path": "/data/openpilot",
|
||||
"casync": {
|
||||
"caibx": f"{OPENPILOT_RELEASES}/{build_metadata.canonical}.caibx"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def create_manifest(target_dir):
|
||||
with open(pathlib.Path(target_dir) / AGNOS_MANIFEST_FILE) as f:
|
||||
agnos_manifest = json.load(f)
|
||||
|
||||
build_metadata = get_build_metadata(args.target_dir)
|
||||
|
||||
return {
|
||||
"build_metadata": dataclasses.asdict(build_metadata),
|
||||
"manifest": [
|
||||
*[create_partition_manifest(entry) for entry in agnos_manifest],
|
||||
create_openpilot_manifest(build_metadata)
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="creates a casync release")
|
||||
parser.add_argument("target_dir", type=str, help="path to a release build of openpilot to create release from")
|
||||
args = parser.parse_args()
|
||||
|
||||
target_dir = pathlib.Path(args.target_dir)
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
casync_dir = pathlib.Path(temp_dir) / "casync"
|
||||
casync_dir.mkdir(parents=True)
|
||||
|
||||
manifest_file = pathlib.Path(temp_dir) / "manifest.json"
|
||||
|
||||
create_casync_caibx(target_dir, casync_dir / "openpilot")
|
||||
upload_casync_release(casync_dir)
|
||||
manifest = create_manifest(target_dir)
|
||||
|
||||
print(json.dumps(manifest, indent=2))
|
||||
Loading…
Reference in New Issue