From 3e9d2b46caa723189820f7f1a837f64a38c6dbea Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 31 Dec 2021 12:38:54 -0800 Subject: [PATCH 01/11] plotjuggler: add mac support --- tools/plotjuggler/README.md | 4 +- tools/plotjuggler/install.sh | 10 ----- tools/plotjuggler/juggle.py | 62 ++++++++++++++++++++------- tools/plotjuggler/test_plotjuggler.py | 3 +- 4 files changed, 48 insertions(+), 31 deletions(-) delete mode 100755 tools/plotjuggler/install.sh diff --git a/tools/plotjuggler/README.md b/tools/plotjuggler/README.md index 8ab150e82148f1..12e23858acf3ed 100644 --- a/tools/plotjuggler/README.md +++ b/tools/plotjuggler/README.md @@ -4,11 +4,9 @@ ## Installation -**NOTE: this is Ubuntu only for now. Pull requests for macOS support are welcome.** - Once you've cloned and are in openpilot, this command will download PlotJuggler and install our plugins: -`cd tools/plotjuggler && ./install.sh` +`cd tools/plotjuggler && ./juggle.py --install` ## Usage diff --git a/tools/plotjuggler/install.sh b/tools/plotjuggler/install.sh deleted file mode 100755 index 27cb0dfb5d5efd..00000000000000 --- a/tools/plotjuggler/install.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -e - -mkdir -p bin -cd bin - -for lib_name in libDataLoadRlog.so libDataStreamCereal.so plotjuggler; do - wget https://github.com/commaai/PlotJuggler/releases/download/latest/${lib_name}.tar.gz - tar -xf ${lib_name}.tar.gz - rm ${lib_name}.tar.gz -done diff --git a/tools/plotjuggler/juggle.py b/tools/plotjuggler/juggle.py index d488750701573a..132df5b82887a6 100755 --- a/tools/plotjuggler/juggle.py +++ b/tools/plotjuggler/juggle.py @@ -2,9 +2,13 @@ import os import sys import multiprocessing +import platform +import shutil import subprocess +import tarfile +import tempfile +import requests import argparse -from tempfile import NamedTemporaryFile from common.basedir import BASEDIR from selfdrive.test.process_replay.compare_logs import save_log @@ -17,6 +21,29 @@ juggle_dir = os.path.dirname(os.path.realpath(__file__)) DEMO_ROUTE = "4cf7a6ad03080c90|2021-09-29--13-46-36" +RELEASES_URL="https://github.com/commaai/PlotJuggler/releases/download/latest" +INSTALL_DIR = os.path.join(juggle_dir, "bin") + + +def install(): + m = f"{platform.system()}-{platform.machine()}" + supported = ("Linux-x86-64", "Darwin-arm64", "Darwin-x86_64") + if m not in supported: + raise Exception(f"Unsupported platform: '{m}'. Supported platforms: {supported}") + + shutil.rmtree(INSTALL_DIR) + os.mkdir(INSTALL_DIR) + + url = os.path.join(RELEASES_URL, m + ".tar.gz") + with requests.get(url, stream=True) as r, tempfile.NamedTemporaryFile() as tmp: + r.raise_for_status() + with open(tmp.name, 'wb') as tmpf: + for chunk in r.iter_content(chunk_size=1024*1024): + tmpf.write(chunk) + + with tarfile.open(tmp.name) as tar: + tar.extractall(path=INSTALL_DIR) + def load_segment(segment_name): print(f"Loading {segment_name}") @@ -29,6 +56,7 @@ def load_segment(segment_name): print(f"Error parsing {segment_name}: {e}") return [] + def start_juggler(fn=None, dbc=None, layout=None): env = os.environ.copy() env["BASEDIR"] = BASEDIR @@ -37,15 +65,15 @@ def start_juggler(fn=None, dbc=None, layout=None): if dbc: env["DBC_NAME"] = dbc - extra_args = [] + extra_args = "" if fn is not None: - extra_args.append(f'-d {fn}') - + extra_args += f"-d {fn}" if layout is not None: - extra_args.append(f'-l {layout}') + extra_args += f"-l {layout}" + + subprocess.call(f'{pj} --plugin_folders {os.path.join(juggle_dir, "bin")} {extra_args}', + shell=True, env=env, cwd=juggle_dir) - extra_args = " ".join(extra_args) - subprocess.call(f'{pj} --plugin_folders {os.path.join(juggle_dir, "bin")} {extra_args}', shell=True, env=env, cwd=juggle_dir) def juggle_route(route_name, segment_number, segment_count, qlog, can, layout): if 'cabana' in route_name: @@ -85,17 +113,18 @@ def juggle_route(route_name, segment_number, segment_count, qlog, can, layout): try: DBC = __import__(f"selfdrive.car.{cp.carParams.carName}.values", fromlist=['DBC']).DBC dbc = DBC[cp.carParams.carFingerprint]['pt'] - except (ImportError, KeyError, AttributeError): + except Exception: pass break - tempfile = NamedTemporaryFile(suffix='.rlog', dir=juggle_dir) + tempfile = tempfile.NamedTemporaryFile(suffix='.rlog', dir=juggle_dir) save_log(tempfile.name, all_data, compress=False) del all_data start_juggler(tempfile.name, dbc, layout) -def get_arg_parser(): + +if __name__ == "__main__": parser = argparse.ArgumentParser(description="A helper to run PlotJuggler on openpilot routes", formatter_class=argparse.ArgumentDefaultsHelpFormatter) @@ -104,17 +133,18 @@ def get_arg_parser(): parser.add_argument("--can", action="store_true", help="Parse CAN data") parser.add_argument("--stream", action="store_true", help="Start PlotJuggler in streaming mode") parser.add_argument("--layout", nargs='?', help="Run PlotJuggler with a pre-defined layout") + parser.add_argument("--install", action="store_true", help="Install or update PlotJuggler + plugins") parser.add_argument("route_name", nargs='?', help="The route name to plot (cabana share URL accepted)") parser.add_argument("segment_number", type=int, nargs='?', help="The index of the segment to plot") parser.add_argument("segment_count", type=int, nargs='?', help="The number of segments to plot", default=1) - return parser - -if __name__ == "__main__": - arg_parser = get_arg_parser() if len(sys.argv) == 1: - arg_parser.print_help() + parser.print_help() + sys.exit() + args = parser.parse_args() + + if args.install: + install() sys.exit() - args = arg_parser.parse_args(sys.argv[1:]) if args.stream: start_juggler(layout=args.layout) diff --git a/tools/plotjuggler/test_plotjuggler.py b/tools/plotjuggler/test_plotjuggler.py index 3e695c19905104..1df7a2afc41aff 100755 --- a/tools/plotjuggler/test_plotjuggler.py +++ b/tools/plotjuggler/test_plotjuggler.py @@ -12,11 +12,10 @@ class TestPlotJuggler(unittest.TestCase): def test_install(self): - exit_code = os.system(os.path.join(BASEDIR, "tools/plotjuggler/install.sh")) + exit_code = os.system(os.path.join(BASEDIR, "tools/plotjuggler/juggle.py")) self.assertEqual(exit_code, 0) def test_run(self): - test_url = get_url("ffccc77938ddbc44|2021-01-04--16-55-41", 0) # Launch PlotJuggler with the executable in the bin directory From 2f0236053b40a664bece8e6b0b12fe84ef67cdd4 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 31 Dec 2021 12:42:30 -0800 Subject: [PATCH 02/11] fix test? --- tools/plotjuggler/test_plotjuggler.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/tools/plotjuggler/test_plotjuggler.py b/tools/plotjuggler/test_plotjuggler.py index 1df7a2afc41aff..c8317f73fd51db 100755 --- a/tools/plotjuggler/test_plotjuggler.py +++ b/tools/plotjuggler/test_plotjuggler.py @@ -8,23 +8,17 @@ from common.basedir import BASEDIR from common.timeout import Timeout from selfdrive.test.openpilotci import get_url +from tools.plotjuggler.juggle import install class TestPlotJuggler(unittest.TestCase): - def test_install(self): - exit_code = os.system(os.path.join(BASEDIR, "tools/plotjuggler/juggle.py")) - self.assertEqual(exit_code, 0) - def test_run(self): - test_url = get_url("ffccc77938ddbc44|2021-01-04--16-55-41", 0) + install() - # Launch PlotJuggler with the executable in the bin directory - os.environ["PLOTJUGGLER_PATH"] = f'{os.path.join(BASEDIR, "tools/plotjuggler/bin/plotjuggler")}' - p = subprocess.Popen(f'QT_QPA_PLATFORM=offscreen {os.path.join(BASEDIR, "tools/plotjuggler/juggle.py")} \ - "{test_url}"', stderr=subprocess.PIPE, shell=True, - start_new_session=True) + p = subprocess.Popen(f'QT_QPA_PLATFORM=offscreen {os.path.join(BASEDIR, "tools/plotjuggler/juggle.py")} --demo', + stderr=subprocess.PIPE, shell=True, start_new_session=True) - # Wait max 60 seconds for the "Done reading Rlog data" signal from the plugin + # Wait for "Done reading Rlog data" signal from the plugin output = "\n" with Timeout(120, error_msg=output): while output.splitlines()[-1] != "Done reading Rlog data": From 41f787baf8f63ad0b42be42ab984cb082c916edc Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 31 Dec 2021 12:43:36 -0800 Subject: [PATCH 03/11] update readme --- tools/plotjuggler/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/plotjuggler/README.md b/tools/plotjuggler/README.md index 12e23858acf3ed..e2370d5a805989 100644 --- a/tools/plotjuggler/README.md +++ b/tools/plotjuggler/README.md @@ -12,7 +12,8 @@ Once you've cloned and are in openpilot, this command will download PlotJuggler ``` $ ./juggle.py -h -usage: juggle.py [-h] [--demo] [--qlog] [--can] [--stream] [--layout [LAYOUT]] [route_name] [segment_number] [segment_count] +usage: juggle.py [-h] [--demo] [--qlog] [--can] [--stream] [--layout [LAYOUT]] [--install] + [route_name] [segment_number] [segment_count] A helper to run PlotJuggler on openpilot routes @@ -28,6 +29,7 @@ optional arguments: --can Parse CAN data (default: False) --stream Start PlotJuggler in streaming mode (default: False) --layout [LAYOUT] Run PlotJuggler with a pre-defined layout (default: None) + --install Install or update PlotJuggler + plugins (default: False) ``` Example: From b430dafdffa66b73da1c3f2ddb527b31a0fbbc1b Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 31 Dec 2021 12:47:06 -0800 Subject: [PATCH 04/11] oops --- tools/plotjuggler/juggle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/plotjuggler/juggle.py b/tools/plotjuggler/juggle.py index 132df5b82887a6..2ef693a9eb9b0d 100755 --- a/tools/plotjuggler/juggle.py +++ b/tools/plotjuggler/juggle.py @@ -27,7 +27,7 @@ def install(): m = f"{platform.system()}-{platform.machine()}" - supported = ("Linux-x86-64", "Darwin-arm64", "Darwin-x86_64") + supported = ("Linux-x86_64", "Darwin-arm64", "Darwin-x86_64") if m not in supported: raise Exception(f"Unsupported platform: '{m}'. Supported platforms: {supported}") From b7c25a95d5644579db6a1584050642cc3b761408 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 31 Dec 2021 12:51:41 -0800 Subject: [PATCH 05/11] fix --- tools/plotjuggler/juggle.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/plotjuggler/juggle.py b/tools/plotjuggler/juggle.py index 2ef693a9eb9b0d..7901008816ba88 100755 --- a/tools/plotjuggler/juggle.py +++ b/tools/plotjuggler/juggle.py @@ -31,7 +31,8 @@ def install(): if m not in supported: raise Exception(f"Unsupported platform: '{m}'. Supported platforms: {supported}") - shutil.rmtree(INSTALL_DIR) + if os.path.exists(INSTASLL_DIR): + shutil.rmtree(INSTALL_DIR) os.mkdir(INSTALL_DIR) url = os.path.join(RELEASES_URL, m + ".tar.gz") From 66b0d78f34207d741ace91a1e5cd38550972d838 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 31 Dec 2021 12:56:11 -0800 Subject: [PATCH 06/11] cleanup --- tools/plotjuggler/juggle.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/plotjuggler/juggle.py b/tools/plotjuggler/juggle.py index 7901008816ba88..c744514d610507 100755 --- a/tools/plotjuggler/juggle.py +++ b/tools/plotjuggler/juggle.py @@ -61,8 +61,7 @@ def load_segment(segment_name): def start_juggler(fn=None, dbc=None, layout=None): env = os.environ.copy() env["BASEDIR"] = BASEDIR - pj = os.getenv("PLOTJUGGLER_PATH", os.path.join(juggle_dir, "bin/plotjuggler")) - + env["PATH"] = f"{INSTALL_DIR}:{os.getenv('PATH', '')}" if dbc: env["DBC_NAME"] = dbc @@ -72,7 +71,7 @@ def start_juggler(fn=None, dbc=None, layout=None): if layout is not None: extra_args += f"-l {layout}" - subprocess.call(f'{pj} --plugin_folders {os.path.join(juggle_dir, "bin")} {extra_args}', + subprocess.call(f'{pj} --plugin_folders {INSTALL_DIR} {extra_args}', shell=True, env=env, cwd=juggle_dir) From 3a72d3e2bed27439df7f5b515731c68e8e6d8872 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 31 Dec 2021 12:57:13 -0800 Subject: [PATCH 07/11] typo --- tools/plotjuggler/juggle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/plotjuggler/juggle.py b/tools/plotjuggler/juggle.py index c744514d610507..857eaa665b5e09 100755 --- a/tools/plotjuggler/juggle.py +++ b/tools/plotjuggler/juggle.py @@ -31,7 +31,7 @@ def install(): if m not in supported: raise Exception(f"Unsupported platform: '{m}'. Supported platforms: {supported}") - if os.path.exists(INSTASLL_DIR): + if os.path.exists(INSTALL_DIR): shutil.rmtree(INSTALL_DIR) os.mkdir(INSTALL_DIR) From c4d567348f0b5b37bfdf6d4fd48b78e1ea767c2e Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 31 Dec 2021 15:26:10 -0800 Subject: [PATCH 08/11] works --- tools/plotjuggler/juggle.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/plotjuggler/juggle.py b/tools/plotjuggler/juggle.py index 857eaa665b5e09..3242217e1f133a 100755 --- a/tools/plotjuggler/juggle.py +++ b/tools/plotjuggler/juggle.py @@ -71,7 +71,7 @@ def start_juggler(fn=None, dbc=None, layout=None): if layout is not None: extra_args += f"-l {layout}" - subprocess.call(f'{pj} --plugin_folders {INSTALL_DIR} {extra_args}', + subprocess.call(f'plotjuggler --plugin_folders {INSTALL_DIR} {extra_args}', shell=True, env=env, cwd=juggle_dir) @@ -117,11 +117,11 @@ def juggle_route(route_name, segment_number, segment_count, qlog, can, layout): pass break - tempfile = tempfile.NamedTemporaryFile(suffix='.rlog', dir=juggle_dir) - save_log(tempfile.name, all_data, compress=False) + tmp = tempfile.NamedTemporaryFile(suffix='.rlog', dir=juggle_dir) + save_log(tmp.name, all_data, compress=False) del all_data - start_juggler(tempfile.name, dbc, layout) + start_juggler(tmp.name, dbc, layout) if __name__ == "__main__": From 64f38393584b58828b1721099496aa799db80cfd Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 31 Dec 2021 15:32:57 -0800 Subject: [PATCH 09/11] little more --- tools/plotjuggler/juggle.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tools/plotjuggler/juggle.py b/tools/plotjuggler/juggle.py index 3242217e1f133a..c9bd1abdd7782d 100755 --- a/tools/plotjuggler/juggle.py +++ b/tools/plotjuggler/juggle.py @@ -47,7 +47,6 @@ def install(): def load_segment(segment_name): - print(f"Loading {segment_name}") if segment_name is None: return [] @@ -76,6 +75,7 @@ def start_juggler(fn=None, dbc=None, layout=None): def juggle_route(route_name, segment_number, segment_count, qlog, can, layout): + # TODO: abstract out the cabana stuff if 'cabana' in route_name: query = parse_qs(urlparse(route_name).query) api = CommaApi(get_token()) @@ -90,8 +90,8 @@ def juggle_route(route_name, segment_number, segment_count, qlog, can, layout): logs = logs[segment_number:segment_number+segment_count] if None in logs: - fallback_answer = input("At least one of the rlogs in this segment does not exist, would you like to use the qlogs? (y/n) : ") - if fallback_answer == 'y': + ans = input(f"{logs.count(None)}/{len(logs)} of the rlogs in this segment are missing, would you like to fall back to the qlogs? (y/n) ") + if ans == 'y': logs = r.qlog_paths() if segment_number is not None: logs = logs[segment_number:segment_number+segment_count] @@ -117,11 +117,10 @@ def juggle_route(route_name, segment_number, segment_count, qlog, can, layout): pass break - tmp = tempfile.NamedTemporaryFile(suffix='.rlog', dir=juggle_dir) - save_log(tmp.name, all_data, compress=False) - del all_data - - start_juggler(tmp.name, dbc, layout) + with tempfile.NamedTemporaryFile(suffix='.rlog', dir=juggle_dir) as tmp: + save_log(tmp.name, all_data, compress=False) + del all_data + start_juggler(tmp.name, dbc, layout) if __name__ == "__main__": From 6b62fa7558fc98875a1caee21f706fadf4e60ffa Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 31 Dec 2021 15:54:52 -0800 Subject: [PATCH 10/11] fix test --- tools/plotjuggler/test_plotjuggler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/plotjuggler/test_plotjuggler.py b/tools/plotjuggler/test_plotjuggler.py index c8317f73fd51db..6559be698d694a 100755 --- a/tools/plotjuggler/test_plotjuggler.py +++ b/tools/plotjuggler/test_plotjuggler.py @@ -7,15 +7,15 @@ from common.basedir import BASEDIR from common.timeout import Timeout -from selfdrive.test.openpilotci import get_url from tools.plotjuggler.juggle import install class TestPlotJuggler(unittest.TestCase): - def test_run(self): + def test_demo(self): install() - p = subprocess.Popen(f'QT_QPA_PLATFORM=offscreen {os.path.join(BASEDIR, "tools/plotjuggler/juggle.py")} --demo', + pj = os.path.join(BASEDIR, "tools/plotjuggler/juggle.py") + p = subprocess.Popen(f'QT_QPA_PLATFORM=offscreen {pj} --demo None 1', stderr=subprocess.PIPE, shell=True, start_new_session=True) # Wait for "Done reading Rlog data" signal from the plugin From 5e82529302fc477c2c0ae6095ccb5113b53ba48e Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Fri, 31 Dec 2021 16:03:11 -0800 Subject: [PATCH 11/11] little faster --- tools/plotjuggler/test_plotjuggler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/plotjuggler/test_plotjuggler.py b/tools/plotjuggler/test_plotjuggler.py index 6559be698d694a..edaec9c80a15b0 100755 --- a/tools/plotjuggler/test_plotjuggler.py +++ b/tools/plotjuggler/test_plotjuggler.py @@ -15,12 +15,12 @@ def test_demo(self): install() pj = os.path.join(BASEDIR, "tools/plotjuggler/juggle.py") - p = subprocess.Popen(f'QT_QPA_PLATFORM=offscreen {pj} --demo None 1', + p = subprocess.Popen(f'QT_QPA_PLATFORM=offscreen {pj} --demo None 1 --qlog', stderr=subprocess.PIPE, shell=True, start_new_session=True) # Wait for "Done reading Rlog data" signal from the plugin output = "\n" - with Timeout(120, error_msg=output): + with Timeout(180, error_msg=output): while output.splitlines()[-1] != "Done reading Rlog data": output += p.stderr.readline().decode("utf-8")