Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

LogReader: try zst on internal source #32751

Merged
merged 14 commits into from
Jun 14, 2024
10 changes: 7 additions & 3 deletions tools/lib/logreader.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,19 +130,23 @@ def valid_file(fn):
return apply_strategy(mode, rlog_paths, qlog_paths, valid_file=valid_file)


def internal_source(sr: SegmentRange, mode: ReadMode) -> LogPaths:
def internal_source(sr: SegmentRange, mode: ReadMode, file_ext: str = "bz2") -> LogPaths:
if not internal_source_available():
raise InternalUnavailableException

def get_internal_url(sr: SegmentRange, seg, file):
return f"cd:/{sr.dongle_id}/{sr.log_id}/{seg}/{file}.bz2"
return f"cd:/{sr.dongle_id}/{sr.log_id}/{seg}/{file}.{file_ext}"

rlog_paths = [get_internal_url(sr, seg, "rlog") for seg in sr.seg_idxs]
qlog_paths = [get_internal_url(sr, seg, "qlog") for seg in sr.seg_idxs]

return apply_strategy(mode, rlog_paths, qlog_paths)


def internal_source_zst(sr: SegmentRange, mode: ReadMode, file_ext: str = "zst") -> LogPaths:
return internal_source(sr, mode, file_ext)
Comment on lines +146 to +147
Copy link
Contributor Author

@sshane sshane Jun 14, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If one or the other doesn't exist, we'll spend ~0.001s extra on one HEAD request to the first log file in the segment range, not bad



def openpilotci_source(sr: SegmentRange, mode: ReadMode) -> LogPaths:
rlog_paths = [get_url(sr.route_name, seg, "rlog") for seg in sr.seg_idxs]
qlog_paths = [get_url(sr.route_name, seg, "qlog") for seg in sr.seg_idxs]
Expand Down Expand Up @@ -175,7 +179,7 @@ def auto_source(sr: SegmentRange, mode=ReadMode.RLOG) -> LogPaths:
if mode == ReadMode.SANITIZED:
return comma_car_segments_source(sr, mode)

SOURCES: list[Source] = [internal_source, openpilotci_source, comma_api_source, comma_car_segments_source,]
SOURCES: list[Source] = [internal_source, internal_source_zst, openpilotci_source, comma_api_source, comma_car_segments_source,]
exceptions = {}

# for automatic fallback modes, auto_source needs to first check if rlogs exist for any source
Expand Down
Loading