delay bz2 compression from logging to uploading (#24392)

* remove log_name

* log without compression

* fix tests

* remove extension for bootlog

* another test fix

* uploader compresses

* also compress in athena

* only compress qlog

* more generic check in do_upload

* fix bootlog compression

* lower loggerd cpu usage

* dont link against bz2

* set core affinity to little cluster

* handle old files
old-commit-hash: 77a6f3d034
This commit is contained in:
Willem Melching
2022-05-03 14:09:17 +02:00
committed by GitHub
parent b35ac8b532
commit be08693d5c
14 changed files with 96 additions and 69 deletions

View File

@@ -112,7 +112,7 @@ class TestEncoder(unittest.TestCase):
# Check encodeIdx
if encode_idx_name is not None:
rlog_path = f"{route_prefix_path}--{i}/rlog.bz2"
rlog_path = f"{route_prefix_path}--{i}/rlog"
msgs = [m for m in LogReader(rlog_path) if m.which() == encode_idx_name]
encode_msgs = [getattr(m, encode_idx_name) for m in msgs]