From 7feae28705842380ec00a6c691b97feeacd3ff3c Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Tue, 26 Oct 2021 23:50:05 -0700 Subject: [PATCH] run onroad tests on release build (#22700) * check startup aelrt * release decorator * run in jenkins * run onroad tests * no push * fix release2 build * fix path * no overwrite * single release build script * make files_eon non empty * files * run in source * add that for now * print * ignore initialzing * print * fetch * run tests last * check alert text * revert some stuff * fixup jenkinsfile Co-authored-by: Comma Device --- Jenkinsfile | 4 +- .../{build_release3.sh => build_release.sh} | 44 ++++++-- release/build_release2.sh | 104 ------------------ release/files_eon | 1 + selfdrive/manager/test/test_manager.py | 0 selfdrive/test/helpers.py | 8 ++ selfdrive/test/test_onroad.py | 24 +++- 7 files changed, 67 insertions(+), 118 deletions(-) rename release/{build_release3.sh => build_release.sh} (67%) delete mode 100755 release/build_release2.sh create mode 100644 release/files_eon mode change 100644 => 100755 selfdrive/manager/test/test_manager.py diff --git a/Jenkinsfile b/Jenkinsfile index dde73b0aa9..0b7b371ab6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -70,7 +70,7 @@ pipeline { } steps { phone_steps("eon-build", [ - ["build release2-staging & dashcam-staging", "cd release && PUSH=1 ./build_release2.sh"], + ["build release2-staging & dashcam-staging", "cd release && PUSH=1 ./build_release.sh"], ]) } } @@ -87,7 +87,7 @@ pipeline { } steps { phone_steps("tici", [ - ["build release3-staging & dashcam3-staging", "PUSH=1 $SOURCE_DIR/release/build_release3.sh"], + ["build release3-staging & dashcam3-staging", "PUSH=1 $SOURCE_DIR/release/build_release.sh"], ]) } } diff --git a/release/build_release3.sh b/release/build_release.sh similarity index 67% rename from release/build_release3.sh rename to release/build_release.sh index 681bc48bb8..20c5206ccc 100755 --- a/release/build_release3.sh +++ b/release/build_release.sh @@ -9,7 +9,17 @@ cd $DIR BUILD_DIR=/data/openpilot SOURCE_DIR="$(git rev-parse --show-toplevel)" -BRANCH=release3-staging +if [ -f /TICI ]; then + FILES_SRC="release/files_tici" + RELEASE_BRANCH=release3-staging + DASHBAM_BRANCH=dashcam3-staging +elif [ -f /EON ]; then + FILES_SRC="release/files_eon" + RELEASE_BRANCH=release2-staging + DASHBAM_BRANCH=dashcam-staging +else + exit 0 +fi # set git identity source $DIR/identity.sh @@ -20,13 +30,14 @@ mkdir -p $BUILD_DIR cd $BUILD_DIR git init git remote add origin git@github.com:commaai/openpilot.git -git checkout -f -B $BRANCH +git fetch origin $RELEASE_BRANCH +git checkout --orphan $RELEASE_BRANCH # do the files copy echo "[-] copying files T=$SECONDS" cd $SOURCE_DIR cp -pR --parents $(cat release/files_common) $BUILD_DIR/ -cp -pR --parents $(cat release/files_tici) $BUILD_DIR/ +cp -pR --parents $(cat $FILES_SRC) $BUILD_DIR/ # in the directory cd $BUILD_DIR @@ -39,6 +50,7 @@ echo "#define COMMA_VERSION \"$VERSION-release\"" > selfdrive/common/version.h echo "[-] committing version $VERSION T=$SECONDS" git add -f . git commit -a -m "openpilot v$VERSION release" +git branch --set-upstream-to=origin/$RELEASE_BRANCH # Build panda firmware pushd panda/ @@ -50,15 +62,20 @@ popd export PYTHONPATH="$BUILD_DIR" scons -j$(nproc) -# Run tests -#python selfdrive/manager/test/test_manager.py -selfdrive/car/tests/test_car_interfaces.py +# Ensure no submodules in release +if test "$(git submodule--helper list | wc -l)" -gt "0"; then + echo "submodules found:" + git submodule--helper list + exit 1 +fi +git submodule status # Cleanup find . -name '*.a' -delete find . -name '*.o' -delete find . -name '*.os' -delete find . -name '*.pyc' -delete +find . -name 'moc_*' -delete find . -name '__pycache__' -delete rm -rf panda/board panda/certs panda/crypto rm -rf .sconsign.dblite Jenkinsfile release/ @@ -78,15 +95,24 @@ touch prebuilt git add -f . git commit --amend -m "openpilot v$VERSION" +# Run tests +TEST_FILES="tools/" +cd $SOURCE_DIR +cp -pR -n --parents $TEST_FILES $BUILD_DIR/ +cd $BUILD_DIR +RELEASE=1 selfdrive/test/test_onroad.py +#selfdrive/manager/test/test_manager.py +selfdrive/car/tests/test_car_interfaces.py +rm -rf $TEST_FILES + if [ ! -z "$PUSH" ]; then echo "[-] pushing T=$SECONDS" - git remote set-url origin git@github.com:commaai/openpilot.git - git push -f origin $BRANCH + git push -f origin $RELEASE_BRANCH # Create dashcam git rm selfdrive/car/*/carcontroller.py git commit -m "create dashcam release from release" - git push -f origin $BRANCH:dashcam3-staging + git push -f origin $RELEASE_BRANCH:$DASHCAM_BRANCH fi echo "[-] done T=$SECONDS" diff --git a/release/build_release2.sh b/release/build_release2.sh deleted file mode 100755 index f7b22792db..0000000000 --- a/release/build_release2.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env bash -set -e - -export GIT_COMMITTER_NAME="Vehicle Researcher" -export GIT_COMMITTER_EMAIL="user@comma.ai" -export GIT_AUTHOR_NAME="Vehicle Researcher" -export GIT_AUTHOR_EMAIL="user@comma.ai" - -export GIT_SSH_COMMAND="ssh -i /data/gitkey" - -# set CLEAN to build outside of CI -if [ ! -z "$CLEAN" ]; then - # Create folders - rm -rf /data/openpilot - mkdir -p /data/openpilot - cd /data/openpilot - - # Create git repo - git init - git remote add origin git@github.com:commaai/openpilot.git - git fetch origin devel-staging -else - cd /data/openpilot - git clean -xdf - git branch -D release2-staging || true -fi - -git fetch origin release2-staging -git fetch origin dashcam-staging - -# Create release2 with no history -if [ ! -z "$CLEAN" ]; then - git checkout --orphan release2-staging origin/devel-staging -else - git checkout --orphan release2-staging -fi - -VERSION=$(cat selfdrive/common/version.h | awk -F[\"-] '{print $2}') -echo "#define COMMA_VERSION \"$VERSION-release\"" > selfdrive/common/version.h - -git commit -m "openpilot v$VERSION" - -# Build signed panda firmware -pushd panda/ -CERT=/tmp/pandaextra/certs/release RELEASE=1 scons -u . -mv board/obj/panda.bin.signed /tmp/panda.bin.signed -popd - -# Build stuff -ln -sfn /data/openpilot /data/pythonpath -export PYTHONPATH="/data/openpilot:/data/openpilot/pyextra" -scons -j3 - -# Run tests -python selfdrive/manager/test/test_manager.py -selfdrive/car/tests/test_car_interfaces.py - -# Ensure no submodules in release -if test "$(git submodule--helper list | wc -l)" -gt "0"; then - echo "submodules found:" - git submodule--helper list - exit 1 -fi -git submodule status - -# Cleanup -find . -name '*.a' -delete -find . -name '*.o' -delete -find . -name '*.os' -delete -find . -name '*.pyc' -delete -find . -name '__pycache__' -delete -rm -rf panda/board panda/certs panda/crypto -rm -rf .sconsign.dblite Jenkinsfile release/ -rm models/supercombo.dlc - -# Move back signed panda fw -mkdir -p panda/board/obj -mv /tmp/panda.bin.signed panda/board/obj/panda.bin.signed - -# Restore third_party -git checkout third_party/ - -# Mark as prebuilt release -touch prebuilt - -# Add built files to git -git add -f . -git commit --amend -m "openpilot v$VERSION" - -# Print committed files that are normally gitignored -#git status --ignored - -if [ ! -z "$PUSH" ]; then - git remote set-url origin git@github.com:commaai/openpilot.git - - # Push to release2-staging - git push -f origin release2-staging - - # Create dashcam release - git rm selfdrive/car/*/carcontroller.py - - git commit -m "create dashcam release from release2" - git push -f origin release2-staging:dashcam-staging -fi diff --git a/release/files_eon b/release/files_eon new file mode 100644 index 0000000000..b43bf86b50 --- /dev/null +++ b/release/files_eon @@ -0,0 +1 @@ +README.md diff --git a/selfdrive/manager/test/test_manager.py b/selfdrive/manager/test/test_manager.py old mode 100644 new mode 100755 diff --git a/selfdrive/test/helpers.py b/selfdrive/test/helpers.py index b227abd168..0bb475c929 100644 --- a/selfdrive/test/helpers.py +++ b/selfdrive/test/helpers.py @@ -1,3 +1,4 @@ +import os import time from functools import wraps @@ -24,6 +25,13 @@ def phone_only(f): f(self, *args, **kwargs) return wrap +def release_only(f): + @wraps(f) + def wrap(self, *args, **kwargs): + if "RELEASE" not in os.environ: + self.skipTest("This test is only for release branches") + f(self, *args, **kwargs) + return wrap def with_processes(processes, init_time=0, ignore_stopped=None): ignore_stopped = [] if ignore_stopped is None else ignore_stopped diff --git a/selfdrive/test/test_onroad.py b/selfdrive/test/test_onroad.py index cbe8776053..a38ff38fcc 100755 --- a/selfdrive/test/test_onroad.py +++ b/selfdrive/test/test_onroad.py @@ -8,14 +8,16 @@ import unittest from collections import Counter from pathlib import Path +from cereal import car import cereal.messaging as messaging from cereal.services import service_list from common.basedir import BASEDIR from common.timeout import Timeout from common.params import Params +from selfdrive.controls.lib.events import EVENTS, ET from selfdrive.hardware import EON, TICI from selfdrive.loggerd.config import ROOT -from selfdrive.test.helpers import set_params_enabled +from selfdrive.test.helpers import set_params_enabled, release_only from tools.lib.logreader import LogReader # Baseline CPU usage by process @@ -56,8 +58,9 @@ if TICI: "./camerad": 31.0, "./_ui": 21.0, "selfdrive.controls.plannerd": 11.7, - "selfdrive.locationd.paramsd": 5.0, "./_dmonitoringmodeld": 10.0, + "selfdrive.locationd.paramsd": 5.0, + "selfdrive.controls.radard": 3.6, "selfdrive.thermald.thermald": 1.5, }) @@ -128,7 +131,7 @@ class TestOnroad(unittest.TestCase): if "DEBUG" in os.environ: segs = filter(lambda x: os.path.exists(os.path.join(x, "rlog.bz2")), Path(ROOT).iterdir()) segs = sorted(segs, key=lambda x: x.stat().st_mtime) - cls.lr = list(LogReader(os.path.join(segs[-2], "rlog.bz2"))) + cls.lr = list(LogReader(os.path.join(segs[-1], "rlog.bz2"))) return os.environ['SKIP_FW_QUERY'] = "1" @@ -171,6 +174,9 @@ class TestOnroad(unittest.TestCase): if proc.wait(60) is None: proc.kill() + cls.lrs = [list(LogReader(os.path.join(str(s), "rlog.bz2"))) for s in cls.segments] + + # use the second segment by default as it's the first full segment cls.lr = list(LogReader(os.path.join(str(cls.segments[1]), "rlog.bz2"))) def test_cloudlog_size(self): @@ -222,5 +228,17 @@ class TestOnroad(unittest.TestCase): print(f" {np.max(np.absolute([np.max(ts)/dt, np.min(ts)/dt]))} {np.std(ts)/dt}") print("="*67) + @release_only + def test_startup(self): + startup_alert = None + for msg in self.lrs[0]: + # can't use carEvents because the first msg can be dropped while loggerd is starting up + if msg.which() == "controlsState": + startup_alert = msg.controlsState.alertText1 + break + expected = EVENTS[car.CarEvent.EventName.startup][ET.PERMANENT].alert_text_1 + self.assertEqual(startup_alert, expected, "wrong startup alert") + + if __name__ == "__main__": unittest.main()