From 50aac48fbab2fd9c556ae3c42cfd9924de40f75c Mon Sep 17 00:00:00 2001 From: Maxime Desroches Date: Wed, 13 Nov 2024 21:27:23 -0800 Subject: [PATCH] ci: faster unit_test (#34019) * multiple * CACHE * ... * cache * now fast * maybe * bp * vv * slow * fast * fix * faster * ruff * info * timeout * info * more * clean * faster * test * collection time * is this real? * fix * back * clean * just to make sure * faster! --- .github/workflows/selfdrive_tests.yaml | 11 +++- cereal/messaging/tests/test_messaging.py | 10 +-- selfdrive/test/process_replay/test_fuzzy.py | 5 +- system/tests/test_logmessaged.py | 6 +- tools/replay/tests/test_replay.cc | 67 --------------------- 5 files changed, 21 insertions(+), 78 deletions(-) diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index 446b0ab44b..fc59d25684 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -173,12 +173,19 @@ jobs: - name: Build openpilot timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache run: ${{ env.RUN }} "scons -j$(nproc)" + - name: Setup cache + uses: ./.github/workflows/auto-cache + with: + path: .ci_cache/comma_download_cache + key: unit_tests_${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }} - name: Run unit tests timeout-minutes: ${{ contains(runner.name, 'nsc') && 1 || 20 }} run: | - ${{ env.RUN }} "MAX_EXAMPLES=1 $PYTEST --timeout 60 -m 'not slow' && \ + ${{ env.RUN }} "$PYTEST --collect-only -m 'not slow' &> /dev/null && \ + MAX_EXAMPLES=1 $PYTEST -m 'not slow' && \ ./selfdrive/ui/tests/create_test_translations.sh && \ - QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations" + QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \ + chmod -R 777 /tmp/comma_download_cache" - name: "Upload coverage to Codecov" uses: codecov/codecov-action@v4 with: diff --git a/cereal/messaging/tests/test_messaging.py b/cereal/messaging/tests/test_messaging.py index 58fa7f7b28..6234a11c08 100644 --- a/cereal/messaging/tests/test_messaging.py +++ b/cereal/messaging/tests/test_messaging.py @@ -168,18 +168,18 @@ class TestMessaging: # this test doesn't work with ZMQ since multiprocessing interrupts it if "ZMQ" not in os.environ: - # wait 15 socket timeouts and make sure it's still retrying + # wait 5 socket timeouts and make sure it's still retrying p = multiprocessing.Process(target=messaging.recv_one_retry, args=(sub_sock,)) p.start() - time.sleep(sock_timeout*15) + time.sleep(sock_timeout*5) assert p.is_alive() p.terminate() - # wait 15 socket timeouts before sending + # wait 5 socket timeouts before sending msg = random_carstate() - delayed_send(sock_timeout*15, pub_sock, msg.to_bytes()) + delayed_send(sock_timeout*5, pub_sock, msg.to_bytes()) start_time = time.monotonic() recvd = messaging.recv_one_retry(sub_sock) - assert (time.monotonic() - start_time) >= sock_timeout*15 + assert (time.monotonic() - start_time) >= sock_timeout*5 assert isinstance(recvd, capnp._DynamicStructReader) assert_carstate(msg.carState, recvd.carState) diff --git a/selfdrive/test/process_replay/test_fuzzy.py b/selfdrive/test/process_replay/test_fuzzy.py index 39e3bfb94a..723112163e 100644 --- a/selfdrive/test/process_replay/test_fuzzy.py +++ b/selfdrive/test/process_replay/test_fuzzy.py @@ -1,4 +1,5 @@ import copy +import os from hypothesis import given, HealthCheck, Phase, settings import hypothesis.strategies as st from parameterized import parameterized @@ -14,13 +15,15 @@ import openpilot.selfdrive.test.process_replay.process_replay as pr NOT_TESTED = ['selfdrived', 'controlsd', 'card', 'plannerd', 'calibrationd', 'dmonitoringd', 'paramsd', 'dmonitoringmodeld', 'modeld'] TEST_CASES = [(cfg.proc_name, copy.deepcopy(cfg)) for cfg in pr.CONFIGS if cfg.proc_name not in NOT_TESTED] +MAX_EXAMPLES = int(os.environ.get("MAX_EXAMPLES", "10")) class TestFuzzProcesses: # TODO: make this faster and increase examples @parameterized.expand(TEST_CASES) @given(st.data()) - @settings(phases=[Phase.generate, Phase.target], max_examples=10, deadline=1000, suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large]) + @settings(phases=[Phase.generate, Phase.target], max_examples=MAX_EXAMPLES, deadline=1000, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large]) def test_fuzz_process(self, proc_name, cfg, data): msgs = FuzzyGenerator.get_random_event_msg(data.draw, events=cfg.pubs, real_floats=True) lr = [log.Event.new_message(**m).as_reader() for m in msgs] diff --git a/system/tests/test_logmessaged.py b/system/tests/test_logmessaged.py index 03c13437bc..9ccc8ef53b 100644 --- a/system/tests/test_logmessaged.py +++ b/system/tests/test_logmessaged.py @@ -19,7 +19,7 @@ class TestLogmessaged: self.error_sock = messaging.sub_sock("logMessage", timeout=1000, conflate=False) # ensure sockets are connected - time.sleep(1) + time.sleep(0.5) messaging.drain_sock(self.sock) messaging.drain_sock(self.error_sock) @@ -35,7 +35,7 @@ class TestLogmessaged: msgs = [f"abc {i}" for i in range(10)] for m in msgs: cloudlog.error(m) - time.sleep(1) + time.sleep(0.5) m = messaging.drain_sock(self.sock) assert len(m) == len(msgs) assert len(self._get_log_files()) >= 1 @@ -45,7 +45,7 @@ class TestLogmessaged: msg = "a"*3*1024*1024 for _ in range(n): cloudlog.info(msg) - time.sleep(1) + time.sleep(0.5) msgs = messaging.drain_sock(self.sock) assert len(msgs) == 0 diff --git a/tools/replay/tests/test_replay.cc b/tools/replay/tests/test_replay.cc index e4d8c3e5f0..6b366169bb 100644 --- a/tools/replay/tests/test_replay.cc +++ b/tools/replay/tests/test_replay.cc @@ -23,42 +23,6 @@ bool download_to_file(const std::string &url, const std::string &local_file, int return false; } -TEST_CASE("httpMultiPartDownload") { - char filename[] = "/tmp/XXXXXX"; - close(mkstemp(filename)); - - const size_t chunk_size = 5 * 1024 * 1024; - std::string content; - SECTION("download to file") { - REQUIRE(download_to_file(TEST_RLOG_URL, filename, chunk_size)); - content = util::read_file(filename); - } - SECTION("download to buffer") { - for (int i = 0; i < 3 && content.empty(); ++i) { - content = httpGet(TEST_RLOG_URL, chunk_size); - std::this_thread::sleep_for(std::chrono::milliseconds(500)); - } - REQUIRE(!content.empty()); - } - REQUIRE(content.size() == 9112651); - REQUIRE(sha256(content) == TEST_RLOG_CHECKSUM); -} - -TEST_CASE("FileReader") { - auto enable_local_cache = GENERATE(true, false); - std::string cache_file = cacheFilePath(TEST_RLOG_URL); - system(("rm " + cache_file + " -f").c_str()); - - FileReader reader(enable_local_cache); - std::string content = reader.read(TEST_RLOG_URL); - REQUIRE(sha256(content) == TEST_RLOG_CHECKSUM); - if (enable_local_cache) { - REQUIRE(sha256(util::read_file(cache_file)) == TEST_RLOG_CHECKSUM); - } else { - REQUIRE(util::file_exists(cache_file) == false); - } -} - TEST_CASE("LogReader") { SECTION("corrupt log") { FileReader reader(true); @@ -134,34 +98,3 @@ std::string download_demo_route() { return data_dir; } - - -TEST_CASE("Getting route") { - std::string data_dir = download_demo_route(); - - auto flags = GENERATE(0, REPLAY_FLAG_QCAMERA); - Route route(DEMO_ROUTE, data_dir); - REQUIRE(route.load()); - REQUIRE(route.segments().size() == 2); - for (int i = 0; i < TEST_REPLAY_SEGMENTS; ++i) { - read_segment(i, route.at(i), flags); - } -} - -TEST_CASE("seek_to") { - QEventLoop loop; - int seek_to = util::random_int(0, 2 * 59); - Replay replay(DEMO_ROUTE, {}, {}, nullptr, REPLAY_FLAG_NO_VIPC); - - QObject::connect(&replay, &Replay::seekedTo, [&](double sec) { - INFO("seek to " << seek_to << "s sought to" << sec); - REQUIRE(sec >= seek_to); - loop.quit(); - }); - - REQUIRE(replay.load()); - replay.start(); - replay.seekTo(seek_to, false); - - loop.exec(); -}