From ecf0a8c8d49dddb18767d929d381f5bbf946c5d8 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 8 Aug 2020 20:59:32 -0700 Subject: [PATCH] Improve CPU usage test reliability (#2002) * run phone tests in parallel * better cpu test * re-enable test * no root * terms version * not one * yes * debug * that's coverd by min cpu percent --- Jenkinsfile | 97 +++++++++++++++++--------------- selfdrive/test/helpers.py | 4 +- selfdrive/test/test_cpu_usage.py | 59 ++++++++----------- 3 files changed, 80 insertions(+), 80 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 0fd2b00a4d..ff8b4a9c01 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -36,16 +36,31 @@ pipeline { } stages { + + stage('Release Build') { + when { + branch 'devel-staging' + } + steps { + phone_steps("eon-build", [ + ["build release2-staging and dashcam-staging", "cd release && PUSH=1 ./build_release2.sh"], + ]) + } + } + stage('openpilot tests') { when { not { anyOf { - branch 'master-ci'; branch 'devel'; branch 'release2'; branch 'release2-staging'; branch 'dashcam'; branch 'dashcam-staging' + branch 'master-ci'; branch 'devel'; branch 'devel-staging'; branch 'release2'; branch 'release2-staging'; branch 'dashcam'; branch 'dashcam-staging' } } } - parallel { + + stages { + + /* stage('PC tests') { agent { dockerfile { @@ -67,6 +82,7 @@ pipeline { } } } + */ stage('On-device Tests') { agent { @@ -75,53 +91,46 @@ pipeline { args '--user=root' } } + stages { + stage('parallel tests') { + parallel { + + stage('Devel Build') { + environment { + CI_PUSH = "${env.BRANCH_NAME == 'master' ? 'master-ci' : ' '}" + } + steps { + phone_steps("eon", [ + ["build devel", "cd release && CI_PUSH=${env.CI_PUSH} ./build_devel.sh"], + ["test openpilot", "nosetests -s selfdrive/test/test_openpilot.py"], + ["test cpu usage", "cd selfdrive/test/ && ./test_cpu_usage.py"], + ["test car interfaces", "cd selfdrive/car/tests/ && ./test_car_interfaces.py"], + ]) + } + } + + stage('Replay Tests') { + steps { + phone_steps("eon2", [ + ["camerad/modeld replay", "cd selfdrive/test/process_replay && ./camera_replay.py"], + ]) + } + } + + stage('HW Tests') { + steps { + phone_steps("eon", [ + ["build cereal", "SCONS_CACHE=1 scons -j4 cereal/"], + ["test sounds", "nosetests -s selfdrive/test/test_sounds.py"], + ["test boardd loopback", "nosetests -s selfdrive/boardd/tests/test_boardd_loopback.py"], + ]) + } + } - stage('Release Build') { - when { - branch 'devel-staging' - } - steps { - phone_steps("eon-build", [ - ["build release2-staging and dashcam-staging", "cd release && PUSH=1 ./build_release2.sh"], - ]) } } - - stage('Devel Build') { - environment { - CI_PUSH = "${env.BRANCH_NAME == 'master' ? 'master-ci' : ' '}" - } - steps { - phone_steps("eon", [ - ["build devel", "cd release && CI_PUSH=${env.CI_PUSH} ./build_devel.sh"], - ["test openpilot", "nosetests -s selfdrive/test/test_openpilot.py"], - //["test cpu usage", "cd selfdrive/test/ && ./test_cpu_usage.py"], - ["test car interfaces", "cd selfdrive/car/tests/ && ./test_car_interfaces.py"], - ]) - } - } - - stage('Replay Tests') { - steps { - phone_steps("eon2", [ - ["camerad/modeld replay", "cd selfdrive/test/process_replay && ./camera_replay.py"], - ]) - } - } - - stage('HW Tests') { - steps { - phone_steps("eon", [ - ["build cereal", "SCONS_CACHE=1 scons -j4 cereal/"], - ["test sounds", "nosetests -s selfdrive/test/test_sounds.py"], - ["test boardd loopback", "nosetests -s selfdrive/boardd/tests/test_boardd_loopback.py"], - ]) - } - } - } - } } diff --git a/selfdrive/test/helpers.py b/selfdrive/test/helpers.py index 733c09dc72..1e53ef4245 100644 --- a/selfdrive/test/helpers.py +++ b/selfdrive/test/helpers.py @@ -5,12 +5,12 @@ from nose.tools import nottest from common.android import ANDROID from common.apk import update_apks, start_offroad, pm_apply_packages, android_packages from common.params import Params -from selfdrive.version import training_version +from selfdrive.version import training_version, terms_version from selfdrive.manager import start_managed_process, kill_managed_process, get_running def set_params_enabled(): params = Params() - params.put("HasAcceptedTerms", "1") + params.put("HasAcceptedTerms", terms_version) params.put("HasCompletedSetup", "1") params.put("OpenpilotEnabledToggle", "1") params.put("CommunityFeaturesToggle", "1") diff --git a/selfdrive/test/test_cpu_usage.py b/selfdrive/test/test_cpu_usage.py index d6d2841c7e..92f8de4ddc 100755 --- a/selfdrive/test/test_cpu_usage.py +++ b/selfdrive/test/test_cpu_usage.py @@ -1,13 +1,12 @@ #!/usr/bin/env python3 +import os import time -import threading -import _thread -import signal import sys +import subprocess import cereal.messaging as messaging +from common.basedir import BASEDIR from common.params import Params -import selfdrive.manager as manager from selfdrive.test.helpers import set_params_enabled def cputime_total(ct): @@ -40,7 +39,7 @@ def print_cpu_usage(first_proc, last_proc): ("./logcatd", 0), ] - r = 0 + r = True dt = (last_proc.logMonoTime - first_proc.logMonoTime) / 1e9 result = "------------------------------------------------\n" for proc_name, normal_cpu_usage in procs: @@ -51,26 +50,25 @@ def print_cpu_usage(first_proc, last_proc): cpu_usage = cpu_time / dt * 100. if cpu_usage > max(normal_cpu_usage * 1.1, normal_cpu_usage + 5.0): result += f"Warning {proc_name} using more CPU than normal\n" - r = 1 + r = False elif cpu_usage < min(normal_cpu_usage * 0.3, max(normal_cpu_usage - 1.0, 0.0)): result += f"Warning {proc_name} using less CPU than normal\n" - r = 1 + r = False result += f"{proc_name.ljust(35)} {cpu_usage:.2f}%\n" except IndexError: result += f"{proc_name.ljust(35)} NO METRICS FOUND\n" - r = 1 + r = False result += "------------------------------------------------\n" print(result) return r -def all_running(): - running = manager.get_running() - return all(p in running and running[p].is_alive() for p in manager.car_started_processes) +def test_cpu_usage(): + cpu_ok = False -return_code = 1 -def test_thread(): + # start manager + manager_path = os.path.join(BASEDIR, "selfdrive/manager.py") + manager_proc = subprocess.Popen(["python", manager_path]) try: - global return_code proc_sock = messaging.sub_sock('procLog', conflate=True, timeout=2000) # wait until everything's started and get first sample @@ -80,33 +78,26 @@ def test_thread(): break time.sleep(2) first_proc = messaging.recv_sock(proc_sock, wait=True) - if first_proc is None or not all_running(): - err_msg = "procLog recv timed out" if first_proc is None else "all car started process not running" - print(f"\n\nTEST FAILED: {err_msg}\n\n") - raise Exception + if first_proc is None: + raise Exception("\n\nTEST FAILED: progLog recv timed out\n\n") # run for a minute and get last sample time.sleep(60) last_proc = messaging.recv_sock(proc_sock, wait=True) - return_code = print_cpu_usage(first_proc, last_proc) - if not all_running(): - return_code = 1 + cpu_ok = print_cpu_usage(first_proc, last_proc) finally: - _thread.interrupt_main() + manager_proc.terminate() + ret = manager_proc.wait(20) + if ret is None: + manager_proc.kill() + return cpu_ok if __name__ == "__main__": - - - # setup signal handler to exit with test status - def handle_exit(sig, frame): - sys.exit(return_code) - signal.signal(signal.SIGINT, handle_exit) - - # start manager and test thread set_params_enabled() Params().delete("CarParams") - t = threading.Thread(target=test_thread) - t.daemon = True - t.start() - manager.main() + passed = False + try: + passed = test_cpu_usage() + finally: + sys.exit(int(not passed))