Skip to content

Commit

Permalink
LogReader: try zst on internal source (#32751)
Browse files Browse the repository at this point in the history
* internal source list files like azure api

* messy but works

* no limit

* simpler

* clean up

* clean up

* clean up

* that's obvious

* better

* we need to unfortunately return a url, so best to take a naive approach for now

* todo

* fix

* clean up
  • Loading branch information
sshane authored Jun 14, 2024
1 parent 68e22fa commit b45caf4
Showing 1 changed file with 7 additions and 3 deletions.
10 changes: 7 additions & 3 deletions tools/lib/logreader.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,19 +130,23 @@ def valid_file(fn):
return apply_strategy(mode, rlog_paths, qlog_paths, valid_file=valid_file)


def internal_source(sr: SegmentRange, mode: ReadMode) -> LogPaths:
def internal_source(sr: SegmentRange, mode: ReadMode, file_ext: str = "bz2") -> LogPaths:
if not internal_source_available():
raise InternalUnavailableException

def get_internal_url(sr: SegmentRange, seg, file):
return f"cd:/{sr.dongle_id}/{sr.log_id}/{seg}/{file}.bz2"
return f"cd:/{sr.dongle_id}/{sr.log_id}/{seg}/{file}.{file_ext}"

rlog_paths = [get_internal_url(sr, seg, "rlog") for seg in sr.seg_idxs]
qlog_paths = [get_internal_url(sr, seg, "qlog") for seg in sr.seg_idxs]

return apply_strategy(mode, rlog_paths, qlog_paths)


def internal_source_zst(sr: SegmentRange, mode: ReadMode, file_ext: str = "zst") -> LogPaths:
return internal_source(sr, mode, file_ext)


def openpilotci_source(sr: SegmentRange, mode: ReadMode) -> LogPaths:
rlog_paths = [get_url(sr.route_name, seg, "rlog") for seg in sr.seg_idxs]
qlog_paths = [get_url(sr.route_name, seg, "qlog") for seg in sr.seg_idxs]
Expand Down Expand Up @@ -175,7 +179,7 @@ def auto_source(sr: SegmentRange, mode=ReadMode.RLOG) -> LogPaths:
if mode == ReadMode.SANITIZED:
return comma_car_segments_source(sr, mode)

SOURCES: list[Source] = [internal_source, openpilotci_source, comma_api_source, comma_car_segments_source,]
SOURCES: list[Source] = [internal_source, internal_source_zst, openpilotci_source, comma_api_source, comma_car_segments_source,]
exceptions = {}

# for automatic fallback modes, auto_source needs to first check if rlogs exist for any source
Expand Down

0 comments on commit b45caf4

Please sign in to comment.