mawei/sunnypilot
1
0
Fork 0
mirror of https://github.com/sunnypilot/sunnypilot.git synced 2026-02-19 01:53:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

qlog_size.py: use zstd (#33137)

Browse Source 
qlog_size: use zstd
This commit is contained in:
Shane Smiskol
2024-07-29 20:13:43 -07:00
committed by GitHub
parent 84cff4fc03
commit 11cb2d3a0b
1 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
selfdrive/debug/internal/qlog_size.py
View File

@@ -1,11 +1,12 @@
#!/usr/bin/env python3
import argparse
import bz2
import zstandard as zstd
from collections import defaultdict
import matplotlib.pyplot as plt
from cereal.services import SERVICE_LIST
from openpilot.system.loggerd.uploader import LOG_COMPRESSION_LEVEL
from openpilot.tools.lib.logreader import LogReader
from tqdm import tqdm
@@ -17,14 +18,14 @@ def make_pie(msgs, typ):
for m in msgs:
msgs_by_type[m.which()].append(m.as_builder().to_bytes())
total = len(bz2.compress(b"".join([m.as_builder().to_bytes() for m in msgs])))
total = len(zstd.compress(b"".join([m.as_builder().to_bytes() for m in msgs]), LOG_COMPRESSION_LEVEL))
uncompressed_total = len(b"".join([m.as_builder().to_bytes() for m in msgs]))
length_by_type = {k: len(b"".join(v)) for k, v in msgs_by_type.items()}
# calculate compressed size by calculating diff when removed from the segment
compressed_length_by_type = {}
for k in tqdm(msgs_by_type.keys(), desc="Compressing"):
compressed_length_by_type[k] = total - len(bz2.compress(b"".join([m.as_builder().to_bytes() for m in msgs if m.which() != k])))
compressed_length_by_type[k] = total - len(zstd.compress(b"".join([m.as_builder().to_bytes() for m in msgs if m.which() != k]), LOG_COMPRESSION_LEVEL))
sizes = sorted(compressed_length_by_type.items(), key=lambda kv: kv[1])