From f33a25594d847c91248b0854adb4eba270d700b1 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Wed, 17 Feb 2021 22:01:08 -0800 Subject: [PATCH 1/5] fallback to qlogs --- tools/lib/route.py | 27 ++++++++++++++++++++------- tools/plotjuggler/juggle.py | 22 ++++++++++++++++++++-- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/tools/lib/route.py b/tools/lib/route.py index 30607415411214..1e2f5a03495157 100644 --- a/tools/lib/route.py +++ b/tools/lib/route.py @@ -11,12 +11,14 @@ EXPLORER_FILE_RE = r'^({})--([a-z]+\.[a-z0-9]+)$'.format(SEGMENT_NAME_RE) OP_SEGMENT_DIR_RE = r'^({})$'.format(SEGMENT_NAME_RE) +QLOG_FILENAMES = ['qlog.bz2'] LOG_FILENAMES = ['rlog.bz2', 'raw_log.bz2'] CAMERA_FILENAMES = ['fcamera.hevc', 'video.hevc'] class Route(object): def __init__(self, route_name, data_dir=None): self.route_name = route_name.replace('_', '|') + self.max_seg_number = 0 if data_dir is not None: self._segments = self._get_segments_local(data_dir) else: @@ -27,14 +29,19 @@ def segments(self): return self._segments def log_paths(self): - max_seg_number = self._segments[-1].canonical_name.segment_num + self.max_seg_number = self._segments[-1].canonical_name.segment_num log_path_by_seg_num = {s.canonical_name.segment_num: s.log_path for s in self._segments} - return [log_path_by_seg_num.get(i, None) for i in range(max_seg_number+1)] + return [log_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)] + + def qlog_paths(self): + self.max_seg_number = self._segments[-1].canonical_name.segment_num + qlog_path_by_seg_num = {s.canonical_name.segment_num: s.qlog_path for s in self._segments} + return [qlog_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)] def camera_paths(self): - max_seg_number = self._segments[-1].canonical_name.segment_num + self.max_seg_number = self._segments[-1].canonical_name.segment_num camera_path_by_seg_num = {s.canonical_name.segment_num: s.camera_path for s in self._segments} - return [camera_path_by_seg_num.get(i, None) for i in range(max_seg_number+1)] + return [camera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)] def _get_segments_remote(self): api = CommaApi(get_token()) @@ -48,12 +55,14 @@ def _get_segments_remote(self): segments[segment_name] = RouteSegment( segment_name, url if fn in LOG_FILENAMES else segments[segment_name].log_path, + url if fn in QLOG_FILENAMES else segments[segment_name].qlog_path, url if fn in CAMERA_FILENAMES else segments[segment_name].camera_path ) else: segments[segment_name] = RouteSegment( segment_name, url if fn in LOG_FILENAMES else None, + url if fn in QLOG_FILENAMES else None, url if fn in CAMERA_FILENAMES else None ) @@ -92,22 +101,26 @@ def _get_segments_local(self, data_dir): log_path = next(path for path, filename in files if filename in LOG_FILENAMES) except StopIteration: log_path = None - + try: + qlog_path = next(path for path, filename in files if filename in QLOG_FILENAMES) + except StopIteration: + qlog_path = None try: camera_path = next(path for path, filename in files if filename in CAMERA_FILENAMES) except StopIteration: camera_path = None - segments.append(RouteSegment(segment, log_path, camera_path)) + segments.append(RouteSegment(segment, log_path, qlog_path, camera_path)) if len(segments) == 0: raise ValueError('Could not find segments for route {} in data directory {}'.format(self.route_name, data_dir)) return sorted(segments, key=lambda seg: seg.canonical_name.segment_num) class RouteSegment(object): - def __init__(self, name, log_path, camera_path): + def __init__(self, name, log_path, qlog_path, camera_path): self._name = RouteSegmentName(name) self.log_path = log_path + self.qlog_path = qlog_path self.camera_path = camera_path @property diff --git a/tools/plotjuggler/juggle.py b/tools/plotjuggler/juggle.py index 1f049b656d971f..6afd3a34ce6d0e 100755 --- a/tools/plotjuggler/juggle.py +++ b/tools/plotjuggler/juggle.py @@ -33,9 +33,23 @@ def juggle_route(route_name): pool = multiprocessing.Pool(24) all_data = [] + missing_rlog = False for d in pool.map(load_segment, r.log_paths()): + if d is None: + missing_rlog = True + break all_data += d + if missing_rlog: + fallback_answer = input("At least one of the rlogs in this segment does not exist, would you like to use the qlogs? (y/n) : ") + if fallback_answer == 'y': + all_data = [] + for d in pool.map(load_segment, r.qlog_paths()): + all_data += d + else: + print("Please try a different route") + return + tempfile = NamedTemporaryFile(suffix='.rlog') save_log(tempfile.name, all_data, compress=False) del all_data @@ -47,8 +61,12 @@ def juggle_segment(route_name, segment_nr): lp = r.log_paths()[segment_nr] if lp is None: - print("This segment does not exist, please try a different one") - return + fallback_answer = input("The rlog for this segment does not exist, would you like to use the qlog? (y/n) : ") + if fallback_answer == 'y': + lp = r.qlog_paths()[segment_nr] + else: + print("Please try a different segment") + return uf = URLFile(lp) juggle_file(uf.name) From 636cd8f1b7cd4988270c98ffac1ea15b51a10dfa Mon Sep 17 00:00:00 2001 From: iejMac Date: Wed, 17 Feb 2021 22:03:43 -0800 Subject: [PATCH 2/5] minor fixes --- tools/lib/route.py | 13 +++++++------ tools/plotjuggler/juggle.py | 8 ++++---- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tools/lib/route.py b/tools/lib/route.py index 1e2f5a03495157..b774850e9c0f6b 100644 --- a/tools/lib/route.py +++ b/tools/lib/route.py @@ -18,28 +18,26 @@ class Route(object): def __init__(self, route_name, data_dir=None): self.route_name = route_name.replace('_', '|') - self.max_seg_number = 0 if data_dir is not None: self._segments = self._get_segments_local(data_dir) else: self._segments = self._get_segments_remote() + self.max_seg_number = self._segments[-1].canonical_name.segment_num + @property def segments(self): return self._segments def log_paths(self): - self.max_seg_number = self._segments[-1].canonical_name.segment_num log_path_by_seg_num = {s.canonical_name.segment_num: s.log_path for s in self._segments} return [log_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)] - + def qlog_paths(self): - self.max_seg_number = self._segments[-1].canonical_name.segment_num qlog_path_by_seg_num = {s.canonical_name.segment_num: s.qlog_path for s in self._segments} return [qlog_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)] def camera_paths(self): - self.max_seg_number = self._segments[-1].canonical_name.segment_num camera_path_by_seg_num = {s.canonical_name.segment_num: s.camera_path for s in self._segments} return [camera_path_by_seg_num.get(i, None) for i in range(self.max_seg_number+1)] @@ -97,14 +95,17 @@ def _get_segments_local(self, data_dir): segments = [] for segment, files in segment_files.items(): + try: log_path = next(path for path, filename in files if filename in LOG_FILENAMES) except StopIteration: log_path = None + try: qlog_path = next(path for path, filename in files if filename in QLOG_FILENAMES) except StopIteration: qlog_path = None + try: camera_path = next(path for path, filename in files if filename in CAMERA_FILENAMES) except StopIteration: @@ -120,7 +121,7 @@ class RouteSegment(object): def __init__(self, name, log_path, qlog_path, camera_path): self._name = RouteSegmentName(name) self.log_path = log_path - self.qlog_path = qlog_path + self.qlog_path = qlog_path self.camera_path = camera_path @property diff --git a/tools/plotjuggler/juggle.py b/tools/plotjuggler/juggle.py index 6afd3a34ce6d0e..381b381bd89994 100755 --- a/tools/plotjuggler/juggle.py +++ b/tools/plotjuggler/juggle.py @@ -74,14 +74,14 @@ def juggle_segment(route_name, segment_nr): def get_arg_parser(): parser = argparse.ArgumentParser(description="PlotJuggler plugin for reading rlogs", formatter_class=argparse.ArgumentDefaultsHelpFormatter) - - parser.add_argument("route_name", nargs='?', help="The name of the route that will be plotted.") + + parser.add_argument("route_name", nargs='?', help="The name of the route that will be plotted.") parser.add_argument("segment_number", type=int, nargs='?', help="The index of the segment that will be plotted") return parser if __name__ == "__main__": - - args = get_arg_parser().parse_args(sys.argv[1:]) + + args = get_arg_parser().parse_args(sys.argv[1:]) if args.segment_number is None: juggle_route(args.route_name) else: From dc9b7cacb11f9cd19fdd44c4dcebe915b3eaadb3 Mon Sep 17 00:00:00 2001 From: iejMac Date: Wed, 17 Feb 2021 22:08:42 -0800 Subject: [PATCH 3/5] removed extra line --- tools/lib/route.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/lib/route.py b/tools/lib/route.py index b774850e9c0f6b..7b65897d6cab87 100644 --- a/tools/lib/route.py +++ b/tools/lib/route.py @@ -24,7 +24,6 @@ def __init__(self, route_name, data_dir=None): self._segments = self._get_segments_remote() self.max_seg_number = self._segments[-1].canonical_name.segment_num - @property def segments(self): return self._segments From 3da9bcb7b041825bc93aa004609e8e25a207f60b Mon Sep 17 00:00:00 2001 From: iejMac Date: Wed, 17 Feb 2021 22:42:59 -0800 Subject: [PATCH 4/5] help on no args --- tools/plotjuggler/juggle.py | 57 +++++++++++++------------------------ 1 file changed, 20 insertions(+), 37 deletions(-) diff --git a/tools/plotjuggler/juggle.py b/tools/plotjuggler/juggle.py index 381b381bd89994..42627691684bf0 100755 --- a/tools/plotjuggler/juggle.py +++ b/tools/plotjuggler/juggle.py @@ -10,7 +10,6 @@ from selfdrive.test.process_replay.compare_logs import save_log from tools.lib.route import Route from tools.lib.logreader import LogReader -from tools.lib.url_file import URLFile def load_segment(segment_name): @@ -26,51 +25,34 @@ def juggle_file(fn): juggle_dir = os.path.dirname(os.path.realpath(__file__)) subprocess.call(f"bin/plotjuggler -d {fn}", shell=True, env=env, cwd=juggle_dir) -def juggle_route(route_name): +def juggle_route(route_name, segment_number): r = Route(route_name) - all_data = [] - - pool = multiprocessing.Pool(24) - all_data = [] - missing_rlog = False - for d in pool.map(load_segment, r.log_paths()): - if d is None: - missing_rlog = True - break - all_data += d + logs = r.log_paths() + if segment_number is not None: + logs = logs[segment_number:segment_number+1] - if missing_rlog: + if None in logs: fallback_answer = input("At least one of the rlogs in this segment does not exist, would you like to use the qlogs? (y/n) : ") if fallback_answer == 'y': - all_data = [] - for d in pool.map(load_segment, r.qlog_paths()): - all_data += d + logs = r.qlog_paths() + if segment_number is not None: + logs = logs[segment_number:segment_number+1] else: - print("Please try a different route") + print(f"Please try a different {'segment' if segment_number is not None else 'route'}") return + all_data = [] + pool = multiprocessing.Pool(24) + for d in pool.map(load_segment, logs): + all_data += d + tempfile = NamedTemporaryFile(suffix='.rlog') save_log(tempfile.name, all_data, compress=False) del all_data juggle_file(tempfile.name) -def juggle_segment(route_name, segment_nr): - r = Route(route_name) - lp = r.log_paths()[segment_nr] - - if lp is None: - fallback_answer = input("The rlog for this segment does not exist, would you like to use the qlog? (y/n) : ") - if fallback_answer == 'y': - lp = r.qlog_paths()[segment_nr] - else: - print("Please try a different segment") - return - - uf = URLFile(lp) - juggle_file(uf.name) - def get_arg_parser(): parser = argparse.ArgumentParser(description="PlotJuggler plugin for reading rlogs", formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -81,8 +63,9 @@ def get_arg_parser(): if __name__ == "__main__": - args = get_arg_parser().parse_args(sys.argv[1:]) - if args.segment_number is None: - juggle_route(args.route_name) - else: - juggle_segment(args.route_name, args.segment_number) + arg_parser = get_arg_parser() + if(len(sys.argv) == 1): + arg_parser.print_help() + sys.exit() + args = arg_parser.parse_args(sys.argv[1:]) + juggle_route(args.route_name, args.segment_number) From 19f010d9c8033b7dfacba3ba11778d97f8a77e05 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Wed, 17 Feb 2021 22:45:52 -0800 Subject: [PATCH 5/5] Update tools/plotjuggler/juggle.py --- tools/plotjuggler/juggle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/plotjuggler/juggle.py b/tools/plotjuggler/juggle.py index 42627691684bf0..70f53b847ba6f4 100755 --- a/tools/plotjuggler/juggle.py +++ b/tools/plotjuggler/juggle.py @@ -64,7 +64,7 @@ def get_arg_parser(): if __name__ == "__main__": arg_parser = get_arg_parser() - if(len(sys.argv) == 1): + if len(sys.argv) == 1: arg_parser.print_help() sys.exit() args = arg_parser.parse_args(sys.argv[1:])