mawei/onepilot
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LogReader: try zst on internal source (#32751)

Browse Source 
* internal source list files like azure api

* messy but works

* no limit

* simpler

* clean up

* clean up

* clean up

* that's obvious

* better

* we need to unfortunately return a url, so best to take a naive approach for now

* todo

* fix

* clean up
old-commit-hash: b45caf403366ed4161516e49b9e433805d9de6c4
This commit is contained in:
Shane Smiskol
2024-06-14 01:12:19 -07:00
committed by GitHub
parent 644156420b
commit 9f50db15c0
1 changed files with 7 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
tools/lib/logreader.py
View File

@@ -130,12 +130,12 @@ def comma_api_source(sr: SegmentRange, mode: ReadMode) -> LogPaths:
return apply_strategy(mode, rlog_paths, qlog_paths, valid_file=valid_file)
def internal_source(sr: SegmentRange, mode: ReadMode) -> LogPaths:
def internal_source(sr: SegmentRange, mode: ReadMode, file_ext: str = "bz2") -> LogPaths:
if not internal_source_available():
raise InternalUnavailableException
def get_internal_url(sr: SegmentRange, seg, file):
return f"cd:/{sr.dongle_id}/{sr.log_id}/{seg}/{file}.bz2"
return f"cd:/{sr.dongle_id}/{sr.log_id}/{seg}/{file}.{file_ext}"
rlog_paths = [get_internal_url(sr, seg, "rlog") for seg in sr.seg_idxs]
qlog_paths = [get_internal_url(sr, seg, "qlog") for seg in sr.seg_idxs]
@@ -143,6 +143,10 @@ def internal_source(sr: SegmentRange, mode: ReadMode) -> LogPaths:
return apply_strategy(mode, rlog_paths, qlog_paths)
def internal_source_zst(sr: SegmentRange, mode: ReadMode, file_ext: str = "zst") -> LogPaths:
return internal_source(sr, mode, file_ext)
def openpilotci_source(sr: SegmentRange, mode: ReadMode) -> LogPaths:
rlog_paths = [get_url(sr.route_name, seg, "rlog") for seg in sr.seg_idxs]
qlog_paths = [get_url(sr.route_name, seg, "qlog") for seg in sr.seg_idxs]
@@ -175,7 +179,7 @@ def auto_source(sr: SegmentRange, mode=ReadMode.RLOG) -> LogPaths:
if mode == ReadMode.SANITIZED:
return comma_car_segments_source(sr, mode)
SOURCES: list[Source] = [internal_source, openpilotci_source, comma_api_source, comma_car_segments_source,]
SOURCES: list[Source] = [internal_source, internal_source_zst, openpilotci_source, comma_api_source, comma_car_segments_source,]
exceptions = {}
# for automatic fallback modes, auto_source needs to first check if rlogs exist for any source