diff --git a/tools/lib/route.py b/tools/lib/route.py index c3bb94dad486db..c87bd04b9ebcdd 100644 --- a/tools/lib/route.py +++ b/tools/lib/route.py @@ -7,9 +7,10 @@ from tools.lib.auth_config import get_token from tools.lib.api import CommaApi -SEGMENT_NAME_RE = r'[a-z0-9]{16}[|_][0-9]{4}-[0-9]{2}-[0-9]{2}--[0-9]{2}-[0-9]{2}-[0-9]{2}--[0-9]+' -EXPLORER_FILE_RE = r'^({})--([a-z]+\.[a-z0-9]+)$'.format(SEGMENT_NAME_RE) -OP_SEGMENT_DIR_RE = r'^({})$'.format(SEGMENT_NAME_RE) +ROUTE_NAME_RE = r'(?P[a-z0-9]{16})[|_/](?P[0-9]{4}-[0-9]{2}-[0-9]{2}--[0-9]{2}-[0-9]{2}-[0-9]{2})' +SEGMENT_NAME_RE = r'{}(?:--|/)(?P[0-9]+)'.format(ROUTE_NAME_RE) +EXPLORER_FILE_RE = r'^(?P{})--(?P[a-z]+\.[a-z0-9]+)$'.format(SEGMENT_NAME_RE) +OP_SEGMENT_DIR_RE = r'^(?P{})$'.format(SEGMENT_NAME_RE) QLOG_FILENAMES = ['qlog.bz2'] QCAMERA_FILENAMES = ['qcamera.ts'] @@ -19,47 +20,51 @@ ECAMERA_FILENAMES = ['ecamera.hevc'] class Route: - def __init__(self, route_name, data_dir=None): + def __init__(self, name, data_dir=None): + self._name = RouteName(name) self.files = None - self.route_name = route_name.replace('_', '|') if data_dir is not None: self._segments = self._get_segments_local(data_dir) else: self._segments = self._get_segments_remote() - self.max_seg_number = self._segments[-1].canonical_name.segment_num + self.max_seg_number = self._segments[-1].name.segment_num + + @property + def name(self): + return self._name @property def segments(self): return self._segments def log_paths(self): - log_path_by_seg_num = {s.canonical_name.segment_num: s.log_path for s in self._segments} + log_path_by_seg_num = {s.name.segment_num: s.log_path for s in self._segments} return [log_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)] def qlog_paths(self): - qlog_path_by_seg_num = {s.canonical_name.segment_num: s.qlog_path for s in self._segments} + qlog_path_by_seg_num = {s.name.segment_num: s.qlog_path for s in self._segments} return [qlog_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)] def camera_paths(self): - camera_path_by_seg_num = {s.canonical_name.segment_num: s.camera_path for s in self._segments} + camera_path_by_seg_num = {s.name.segment_num: s.camera_path for s in self._segments} return [camera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)] def dcamera_paths(self): - dcamera_path_by_seg_num = {s.canonical_name.segment_num: s.dcamera_path for s in self._segments} + dcamera_path_by_seg_num = {s.name.segment_num: s.dcamera_path for s in self._segments} return [dcamera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)] def ecamera_paths(self): - ecamera_path_by_seg_num = {s.canonical_name.segment_num: s.ecamera_path for s in self._segments} + ecamera_path_by_seg_num = {s.name.segment_num: s.ecamera_path for s in self._segments} return [ecamera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)] def qcamera_paths(self): - qcamera_path_by_seg_num = {s.canonical_name.segment_num: s.qcamera_path for s in self._segments} + qcamera_path_by_seg_num = {s.name.segment_num: s.qcamera_path for s in self._segments} return [qcamera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)] # TODO: refactor this, it's super repetitive def _get_segments_remote(self): api = CommaApi(get_token()) - route_files = api.get('v1/route/' + self.route_name + '/files') + route_files = api.get('v1/route/' + self.name.canonical_name + '/files') self.files = list(chain.from_iterable(route_files.values())) segments = {} @@ -67,7 +72,7 @@ def _get_segments_remote(self): _, dongle_id, time_str, segment_num, fn = urlparse(url).path.rsplit('/', maxsplit=4) segment_name = f'{dongle_id}|{time_str}--{segment_num}' if segments.get(segment_name): - segments[segment_name] = RouteSegment( + segments[segment_name] = Segment( segment_name, url if fn in LOG_FILENAMES else segments[segment_name].log_path, url if fn in QLOG_FILENAMES else segments[segment_name].qlog_path, @@ -77,7 +82,7 @@ def _get_segments_remote(self): url if fn in QCAMERA_FILENAMES else segments[segment_name].qcamera_path, ) else: - segments[segment_name] = RouteSegment( + segments[segment_name] = Segment( segment_name, url if fn in LOG_FILENAMES else None, url if fn in QLOG_FILENAMES else None, @@ -87,7 +92,7 @@ def _get_segments_remote(self): url if fn in QCAMERA_FILENAMES else None, ) - return sorted(segments.values(), key=lambda seg: seg.canonical_name.segment_num) + return sorted(segments.values(), key=lambda seg: seg.name.segment_num) def _get_segments_local(self, data_dir): files = os.listdir(data_dir) @@ -99,20 +104,21 @@ def _get_segments_local(self, data_dir): op_match = re.match(OP_SEGMENT_DIR_RE, f) if explorer_match: - segment_name, fn = explorer_match.groups() - if segment_name.replace('_', '|').startswith(self.route_name): + segment_name = explorer_match.group('segment_name') + fn = explorer_match.group('file_name') + if segment_name.replace('_', '|').startswith(self.name.canonical_name): segment_files[segment_name].append((fullpath, fn)) elif op_match and os.path.isdir(fullpath): - segment_name, = op_match.groups() - if segment_name.startswith(self.route_name): + segment_name = op_match.group('segment_name') + if segment_name.startswith(self.name.canonical_name): for seg_f in os.listdir(fullpath): segment_files[segment_name].append((os.path.join(fullpath, seg_f), seg_f)) - elif f == self.route_name: + elif f == self.name.canonical_name: for seg_num in os.listdir(fullpath): if not seg_num.isdigit(): continue - segment_name = f'{self.route_name}--{seg_num}' + segment_name = f'{self.name.canonical_name}--{seg_num}' for seg_f in os.listdir(os.path.join(fullpath, seg_num)): segment_files[segment_name].append((os.path.join(fullpath, seg_num, seg_f), seg_f)) @@ -149,15 +155,15 @@ def _get_segments_local(self, data_dir): except StopIteration: qcamera_path = None - segments.append(RouteSegment(segment, log_path, qlog_path, camera_path, dcamera_path, ecamera_path, qcamera_path)) + segments.append(Segment(segment, log_path, qlog_path, camera_path, dcamera_path, ecamera_path, qcamera_path)) if len(segments) == 0: - raise ValueError(f'Could not find segments for route {self.route_name} in data directory {data_dir}') - return sorted(segments, key=lambda seg: seg.canonical_name.segment_num) + raise ValueError(f'Could not find segments for route {self.name.canonical_name} in data directory {data_dir}') + return sorted(segments, key=lambda seg: seg.name.segment_num) -class RouteSegment: +class Segment: def __init__(self, name, log_path, qlog_path, camera_path, dcamera_path, ecamera_path, qcamera_path): - self._name = RouteSegmentName(name) + self._name = SegmentName(name) self.log_path = log_path self.qlog_path = qlog_path self.camera_path = camera_path @@ -167,21 +173,55 @@ def __init__(self, name, log_path, qlog_path, camera_path, dcamera_path, ecamera @property def name(self): - return str(self._name) + return self._name + +class RouteName: + def __init__(self, name_str: str): + self._name_str = name_str + delim = next(c for c in self._name_str if c in ("|", "/")) + self._dongle_id, self._time_str = self._name_str.split(delim) + + assert len(self._dongle_id) == 16, self._name_str + assert len(self._time_str) == 20, self._name_str + self._canonical_name = f"{self._dongle_id}|{self._time_str}" @property - def canonical_name(self): - return self._name + def canonical_name(self) -> str: return self._canonical_name + + @property + def dongle_id(self) -> str: return self._dongle_id -class RouteSegmentName: - def __init__(self, name_str): - self._segment_name_str = name_str - self._route_name_str, num_str = self._segment_name_str.rsplit("--", 1) - self._num = int(num_str) + @property + def time_str(self) -> str: return self._time_str + + def __str__(self) -> str: return self._canonical_name + +class SegmentName: + # TODO: add constructor that takes dongle_id, time_str, segment_num and then create instances + # of this class instead of manually constructing a segment name (use canonical_name prop instead) + def __init__(self, name_str: str, allow_route_name=False): + self._name_str = name_str + seg_num_delim = "--" if self._name_str.count("--") == 2 else "/" + name_parts = self._name_str.rsplit(seg_num_delim, 1) + if allow_route_name and len(name_parts) == 1: + name_parts.append("-1") # no segment number + self._route_name = RouteName(name_parts[0]) + self._num = int(name_parts[1]) + self._canonical_name = f"{self._route_name._dongle_id}|{self._route_name._time_str}--{self._num}" + + @property + def canonical_name(self) -> str: return self._canonical_name + + @property + def dongle_id(self) -> str: return self._route_name.dongle_id + + @property + def time_str(self) -> str: return self._route_name.time_str + + @property + def segment_num(self) -> int: return self._num @property - def segment_num(self): - return self._num + def route_name(self) -> RouteName: return self._route_name - def __str__(self): - return self._segment_name_str + def __str__(self) -> str: return self._canonical_name diff --git a/tools/plotjuggler/README.md b/tools/plotjuggler/README.md index e2370d5a805989..2e79aecbcacd81 100644 --- a/tools/plotjuggler/README.md +++ b/tools/plotjuggler/README.md @@ -13,14 +13,14 @@ Once you've cloned and are in openpilot, this command will download PlotJuggler ``` $ ./juggle.py -h usage: juggle.py [-h] [--demo] [--qlog] [--can] [--stream] [--layout [LAYOUT]] [--install] - [route_name] [segment_number] [segment_count] + [route_or_segment_name] [segment_count] A helper to run PlotJuggler on openpilot routes positional arguments: - route_name The route name to plot (cabana share URL accepted) (default: None) - segment_number The index of the segment to plot (default: None) - segment_count The number of segments to plot (default: 1) + route_or_segment_name + The route or segment name to plot (cabana share URL accepted) (default: None) + segment_count The number of segments to plot (default: None) optional arguments: -h, --help show this help message and exit @@ -32,10 +32,14 @@ optional arguments: --install Install or update PlotJuggler + plugins (default: False) ``` -Example: +Examples using route name: `./juggle.py "4cf7a6ad03080c90|2021-09-29--13-46-36"` +Examples using segment name: + +`./juggle.py "4cf7a6ad03080c90|2021-09-29--13-46-36--1"` + ## Streaming Explore live data from your car! Follow these steps to stream from your comma device to your laptop: diff --git a/tools/plotjuggler/juggle.py b/tools/plotjuggler/juggle.py index 3683a6a750dfe8..7cafaf60acf8e0 100755 --- a/tools/plotjuggler/juggle.py +++ b/tools/plotjuggler/juggle.py @@ -15,7 +15,7 @@ from tools.lib.api import CommaApi from tools.lib.auth_config import get_token from tools.lib.robust_logreader import RobustLogReader -from tools.lib.route import Route +from tools.lib.route import Route, SegmentName from urllib.parse import urlparse, parse_qs juggle_dir = os.path.dirname(os.path.realpath(__file__)) @@ -74,29 +74,33 @@ def start_juggler(fn=None, dbc=None, layout=None): subprocess.call(cmd, shell=True, env=env, cwd=juggle_dir) -def juggle_route(route_name, segment_number, segment_count, qlog, can, layout): - # TODO: abstract out the cabana stuff - if 'cabana' in route_name: - query = parse_qs(urlparse(route_name).query) +def juggle_route(route_or_segment_name, segment_count, qlog, can, layout): + segment_start = 0 + if 'cabana' in route_or_segment_name: + query = parse_qs(urlparse(route_or_segment_name).query) api = CommaApi(get_token()) logs = api.get(f'v1/route/{query["route"][0]}/log_urls?sig={query["sig"][0]}&exp={query["exp"][0]}') - elif route_name.startswith("http://") or route_name.startswith("https://") or os.path.isfile(route_name): - logs = [route_name] + elif route_or_segment_name.startswith("http://") or route_or_segment_name.startswith("https://") or os.path.isfile(route_or_segment_name): + logs = [route_or_segment_name] else: - r = Route(route_name) + route_or_segment_name = SegmentName(route_or_segment_name, allow_route_name=True) + segment_start = max(route_or_segment_name.segment_num, 0) + + if route_or_segment_name.segment_num != -1 and segment_count is None: + segment_count = 1 + + r = Route(route_or_segment_name.route_name.canonical_name) logs = r.qlog_paths() if qlog else r.log_paths() - if segment_number is not None: - logs = logs[segment_number:segment_number+segment_count] + segment_end = segment_start + segment_count if segment_count else -1 + logs = logs[segment_start:segment_end] if None in logs: ans = input(f"{logs.count(None)}/{len(logs)} of the rlogs in this segment are missing, would you like to fall back to the qlogs? (y/n) ") if ans == 'y': - logs = r.qlog_paths() - if segment_number is not None: - logs = logs[segment_number:segment_number+segment_count] + logs = r.qlog_paths()[segment_start:segment_end] else: - print(f"Please try a different {'segment' if segment_number is not None else 'route'}") + print("Please try a different route or segment") return all_data = [] @@ -133,9 +137,9 @@ def juggle_route(route_name, segment_number, segment_count, qlog, can, layout): parser.add_argument("--stream", action="store_true", help="Start PlotJuggler in streaming mode") parser.add_argument("--layout", nargs='?', help="Run PlotJuggler with a pre-defined layout") parser.add_argument("--install", action="store_true", help="Install or update PlotJuggler + plugins") - parser.add_argument("route_name", nargs='?', help="The route name to plot (cabana share URL accepted)") - parser.add_argument("segment_number", type=int, nargs='?', help="The index of the segment to plot") - parser.add_argument("segment_count", type=int, nargs='?', help="The number of segments to plot", default=1) + parser.add_argument("route_or_segment_name", nargs='?', help="The route or segment name to plot (cabana share URL accepted)") + parser.add_argument("segment_count", type=int, nargs='?', help="The number of segments to plot") + if len(sys.argv) == 1: parser.print_help() sys.exit() @@ -148,5 +152,5 @@ def juggle_route(route_name, segment_number, segment_count, qlog, can, layout): if args.stream: start_juggler(layout=args.layout) else: - route = DEMO_ROUTE if args.demo else args.route_name.strip() - juggle_route(route, args.segment_number, args.segment_count, args.qlog, args.can, args.layout) + route_or_segment_name = DEMO_ROUTE if args.demo else args.route_or_segment_name.strip() + juggle_route(route_or_segment_name, args.segment_count, args.qlog, args.can, args.layout)