From f4cca5c9db4f27e0718cdf58e3d0871dbd6b05d3 Mon Sep 17 00:00:00 2001 From: whyekit-aptiv Date: Tue, 7 Jul 2020 10:50:07 +0800 Subject: [PATCH 01/17] Initial commit to address comments --- python-sdk/nuscenes/nuscenes.py | 16 +- python-sdk/nuscenes/utils/color_map.py | 3 +- .../nuscenes_lidarseg_tutorial.ipynb | 214 +++++++++++++++++- 3 files changed, 216 insertions(+), 17 deletions(-) diff --git a/python-sdk/nuscenes/nuscenes.py b/python-sdk/nuscenes/nuscenes.py index e0cd50b6..250ca318 100644 --- a/python-sdk/nuscenes/nuscenes.py +++ b/python-sdk/nuscenes/nuscenes.py @@ -472,8 +472,8 @@ def get_sample_lidarseg_stats(self, sample_token: str, sort_counts: bool = True, def list_categories(self) -> None: self.explorer.list_categories() - def list_lidarseg_categories(self) -> None: - self.explorer.list_lidarseg_categories() + def list_lidarseg_categories(self, sort_counts: bool = True) -> None: + self.explorer.list_lidarseg_categories(sort_counts=sort_counts) def list_attributes(self) -> None: self.explorer.list_attributes() @@ -624,10 +624,12 @@ def list_categories(self) -> None: np.mean(stats[:, 2]), np.std(stats[:, 2]), np.mean(stats[:, 3]), np.std(stats[:, 3]))) - def list_lidarseg_categories(self) -> None: + def list_lidarseg_categories(self, sort_counts: bool = True) -> None: """ Print categories and counts of the lidarseg data. These stats only cover the split specified in nusc.version. + :param sort_counts: If True, the stats will be printed in ascending order of frequency; if False, + the stats will be printed alphabetically according to class name. """ assert hasattr(self.nusc, 'lidarseg'), 'Error: nuScenes-lidarseg not installed!' @@ -650,11 +652,15 @@ def list_lidarseg_categories(self) -> None: for i in range(len(lidarseg_counts)): lidarseg_counts_dict[self.nusc.lidarseg_idx2name_mapping[i]] = lidarseg_counts[i] - out = sorted(lidarseg_counts_dict.items(), key=lambda item: item[1]) + if sort_counts: + out = sorted(lidarseg_counts_dict.items(), key=lambda item: item[1]) + else: + out = sorted(lidarseg_counts_dict.items()) + # Print frequency counts of each class in the lidarseg dataset. for class_name, count in out: idx = get_key_from_value(self.nusc.lidarseg_idx2name_mapping, class_name) - print('{:3} {:35} nbr_points={:12,}'.format(idx, class_name, count)) + print('{:3} {:40} nbr_points={:12,}'.format(idx, class_name, count)) print('Calculated stats for {} point clouds in {:.1f} seconds.\n====='.format( len(self.nusc.lidarseg), time.time() - start_time)) diff --git a/python-sdk/nuscenes/utils/color_map.py b/python-sdk/nuscenes/utils/color_map.py index ff3b3c37..a08a883f 100644 --- a/python-sdk/nuscenes/utils/color_map.py +++ b/python-sdk/nuscenes/utils/color_map.py @@ -38,7 +38,8 @@ def get_colormap() -> Dict[str, Iterable[int]]: "flat.other": [175, 0, 75], "static.manmade": [222, 184, 135], # Burlywood "static.vegetation": [0, 175, 0], # Green - "static.other": [255, 228, 196] # Bisque + "static.other": [255, 228, 196], # Bisque + "vehicle.ego": [255, 240, 245] } return classname_to_color diff --git a/python-sdk/tutorials/nuscenes_lidarseg_tutorial.ipynb b/python-sdk/tutorials/nuscenes_lidarseg_tutorial.ipynb index e477a6f2..fd9c18e8 100644 --- a/python-sdk/tutorials/nuscenes_lidarseg_tutorial.ipynb +++ b/python-sdk/tutorials/nuscenes_lidarseg_tutorial.ipynb @@ -16,7 +16,7 @@ "metadata": {}, "source": [ "## Setup\n", - "To install the nuScenes-lidarseg expansion, please download the dataset from https://www.nuscenes.org/download. Unpack the compressed file(s) into `/data/sets/nuscenes` and your folder structure should end up looking like this:\n", + "To install the nuScenes-lidarseg expansion, download the dataset from https://www.nuscenes.org/download. Unpack the compressed file(s) into `/data/sets/nuscenes` and your folder structure should end up looking like this:\n", "```\n", "└── nuscenes \n", " ├── Usual nuscenes folders (i.e. samples, sweep)\n", @@ -43,12 +43,43 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "======\n", + "Loading NuScenes tables for version v1.0-mini...\n", + "Loading nuScenes-lidarseg...\n", + "33 category,\n", + "8 attribute,\n", + "4 visibility,\n", + "911 instance,\n", + "12 sensor,\n", + "120 calibrated_sensor,\n", + "31206 ego_pose,\n", + "8 log,\n", + "10 scene,\n", + "404 sample,\n", + "31206 sample_data,\n", + "18538 sample_annotation,\n", + "4 map,\n", + "404 lidarseg,\n", + "Done loading in 0.4 seconds.\n", + "======\n", + "Reverse indexing ...\n", + "Done reverse indexing in 0.1 seconds.\n", + "======\n" + ] + } + ], "source": [ "%matplotlib inline\n", "\n", + "import sys, os\n", + "sys.path.insert(0, os.path.expanduser('~/Desktop/nuscenes-devkit/python-sdk'))\n", "from nuscenes import NuScenes\n", "\n", "nusc = NuScenes(version='v1.0-mini', dataroot='/data/sets/nuscenes', verbose=True)" @@ -66,16 +97,177 @@ "metadata": {}, "source": [ "## Statistics of lidarseg dataset for the v1.0-mini split\n", - "Let's get a quick feel of the lidarseg dataset by looking at what classes are in it and the number of points belonging to each class. The classes will be sorted in ascending order based on the number of points." + "Let's get a quick feel of the lidarseg dataset by looking at what classes are in it and the number of points belonging to each class. The classes will be sorted in ascending order based on the number of points (since `sort_counts=True` below; if you wish to sort the statistics by alphabetical order, just set `sort_counts=False`)." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Calculating stats for nuScenes-lidarseg...\n", + " 3 human.pedestrian.wheelchair nbr_points= 0\n", + " 4 human.pedestrian.stroller nbr_points= 0\n", + " 8 animal nbr_points= 0\n", + " 16 vehicle.emergency.ambulance nbr_points= 0\n", + " 17 vehicle.emergency.police nbr_points= 0\n", + " 31 vehicle.ego nbr_points= 0\n", + " 32 vehicle.emergency.firetruck nbr_points= 0\n", + " 22 movable_object.debris nbr_points= 48\n", + " 6 human.pedestrian.police_officer nbr_points= 64\n", + " 2 human.pedestrian.child nbr_points= 230\n", + " 7 human.pedestrian.construction_worker nbr_points= 1,412\n", + " 11 vehicle.bicycle nbr_points= 1,463\n", + " 21 movable_object.pushable_pullable nbr_points= 2,293\n", + " 5 human.pedestrian.personal_mobility nbr_points= 4,096\n", + " 23 static_object.bicycle_rack nbr_points= 4,476\n", + " 20 movable_object.trafficcone nbr_points= 6,206\n", + " 10 vehicle.motorcycle nbr_points= 6,713\n", + " 0 noise nbr_points= 12,561\n", + " 18 vehicle.trailer nbr_points= 12,787\n", + " 13 vehicle.bus.rigid nbr_points= 29,694\n", + " 15 vehicle.construction nbr_points= 39,300\n", + " 12 vehicle.bus.bendy nbr_points= 40,536\n", + " 1 human.pedestrian.adult nbr_points= 43,812\n", + " 19 movable_object.barrier nbr_points= 55,298\n", + " 27 flat.other nbr_points= 150,153\n", + " 14 vehicle.truck nbr_points= 304,234\n", + " 9 vehicle.car nbr_points= 521,237\n", + " 26 flat.terrain nbr_points= 696,526\n", + " 25 flat.sidewalk nbr_points= 746,980\n", + " 29 static.vegetation nbr_points= 1,565,272\n", + " 28 static.manmade nbr_points= 2,067,510\n", + " 30 static.other nbr_points= 3,643,428\n", + " 24 flat.driveable_surface nbr_points= 4,069,879\n", + "Calculated stats for 404 point clouds in 0.1 seconds.\n", + "=====\n" + ] + } + ], + "source": [ + "nusc.list_lidarseg_categories(sort_counts=True)" + ] + }, + { + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "nusc.list_lidarseg_categories()" + "With `list_lidarseg_categories`, you can get the index which each class name belongs to by looking at the leftmost column. You can also get a mapping of the indices to the class names from the `lidarseg_idx2name_mapping` attribute of the NuScenes class." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 'noise',\n", + " 1: 'human.pedestrian.adult',\n", + " 2: 'human.pedestrian.child',\n", + " 3: 'human.pedestrian.wheelchair',\n", + " 4: 'human.pedestrian.stroller',\n", + " 5: 'human.pedestrian.personal_mobility',\n", + " 6: 'human.pedestrian.police_officer',\n", + " 7: 'human.pedestrian.construction_worker',\n", + " 8: 'animal',\n", + " 9: 'vehicle.car',\n", + " 10: 'vehicle.motorcycle',\n", + " 11: 'vehicle.bicycle',\n", + " 12: 'vehicle.bus.bendy',\n", + " 13: 'vehicle.bus.rigid',\n", + " 14: 'vehicle.truck',\n", + " 15: 'vehicle.construction',\n", + " 16: 'vehicle.emergency.ambulance',\n", + " 17: 'vehicle.emergency.police',\n", + " 18: 'vehicle.trailer',\n", + " 19: 'movable_object.barrier',\n", + " 20: 'movable_object.trafficcone',\n", + " 21: 'movable_object.pushable_pullable',\n", + " 22: 'movable_object.debris',\n", + " 23: 'static_object.bicycle_rack',\n", + " 24: 'flat.driveable_surface',\n", + " 25: 'flat.sidewalk',\n", + " 26: 'flat.terrain',\n", + " 27: 'flat.other',\n", + " 28: 'static.manmade',\n", + " 29: 'static.vegetation',\n", + " 30: 'static.other',\n", + " 31: 'vehicle.ego',\n", + " 32: 'vehicle.emergency.firetruck'}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nusc.lidarseg_idx2name_mapping" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Conversely, you can get the mapping of the class names to the indices from the `lidarseg_name2idx_mapping` attribute of the NuScenes class." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'noise': 0,\n", + " 'human.pedestrian.adult': 1,\n", + " 'human.pedestrian.child': 2,\n", + " 'human.pedestrian.wheelchair': 3,\n", + " 'human.pedestrian.stroller': 4,\n", + " 'human.pedestrian.personal_mobility': 5,\n", + " 'human.pedestrian.police_officer': 6,\n", + " 'human.pedestrian.construction_worker': 7,\n", + " 'animal': 8,\n", + " 'vehicle.car': 9,\n", + " 'vehicle.motorcycle': 10,\n", + " 'vehicle.bicycle': 11,\n", + " 'vehicle.bus.bendy': 12,\n", + " 'vehicle.bus.rigid': 13,\n", + " 'vehicle.truck': 14,\n", + " 'vehicle.construction': 15,\n", + " 'vehicle.emergency.ambulance': 16,\n", + " 'vehicle.emergency.police': 17,\n", + " 'vehicle.trailer': 18,\n", + " 'movable_object.barrier': 19,\n", + " 'movable_object.trafficcone': 20,\n", + " 'movable_object.pushable_pullable': 21,\n", + " 'movable_object.debris': 22,\n", + " 'static_object.bicycle_rack': 23,\n", + " 'flat.driveable_surface': 24,\n", + " 'flat.sidewalk': 25,\n", + " 'flat.terrain': 26,\n", + " 'flat.other': 27,\n", + " 'static.manmade': 28,\n", + " 'static.vegetation': 29,\n", + " 'static.other': 30,\n", + " 'vehicle.ego': 31,\n", + " 'vehicle.emergency.firetruck': 32}" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "nusc.lidarseg_name2idx_mapping" ] }, { @@ -83,7 +275,7 @@ "metadata": {}, "source": [ "## Pick a sample token\n", - "Let's pick a sample token to use for this tutorial." + "Let's pick a sample to use for this tutorial." ] }, { @@ -100,7 +292,7 @@ "metadata": {}, "source": [ "## Get statistics of a lidarseg sample token\n", - "Let's pick a lidarseg sample token and take a look at what objects are present in this particular sample." + "Now let's take a look at what classes are present in the pointcloud of this particular sample." ] }, { @@ -403,7 +595,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.6" + "version": "3.7.7" } }, "nbformat": 4, From 5ed33de32b4feab3e63508d784aedca95ca0b759 Mon Sep 17 00:00:00 2001 From: whyekit-aptiv Date: Tue, 7 Jul 2020 11:53:29 +0800 Subject: [PATCH 02/17] Fix bug where passing np.array instead of array into filter_lidarseg_labels throws an error --- python-sdk/nuscenes/nuscenes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python-sdk/nuscenes/nuscenes.py b/python-sdk/nuscenes/nuscenes.py index 250ca318..61448103 100644 --- a/python-sdk/nuscenes/nuscenes.py +++ b/python-sdk/nuscenes/nuscenes.py @@ -822,7 +822,7 @@ def map_pointcloud_to_image(self, # later use. colors = colormap_to_colors(self.nusc.colormap, self.nusc.lidarseg_name2idx_mapping) - if filter_lidarseg_labels: + if filter_lidarseg_labels is not None: colors = filter_colors(colors, filter_lidarseg_labels) coloring = colors[points_label] else: @@ -1226,7 +1226,7 @@ def render_sample_data(self, # appropriate format. coloring = colormap_to_colors(self.nusc.colormap, self.nusc.lidarseg_name2idx_mapping) - if filter_lidarseg_labels: + if filter_lidarseg_labels is not None: coloring = filter_colors(coloring, filter_lidarseg_labels) colors = coloring[points_label] else: From 82f8e7b341fa74aadf6c6acb84edfe28e4382da6 Mon Sep 17 00:00:00 2001 From: whyekit-aptiv Date: Tue, 7 Jul 2020 15:12:48 +0800 Subject: [PATCH 03/17] Rephrase some parts of tutorial --- .../nuscenes/lidarseg/lidarseg_utils.py | 3 + .../nuscenes_lidarseg_tutorial.ipynb | 200 +++--------------- 2 files changed, 29 insertions(+), 174 deletions(-) diff --git a/python-sdk/nuscenes/lidarseg/lidarseg_utils.py b/python-sdk/nuscenes/lidarseg/lidarseg_utils.py index f9036a2e..c063df38 100644 --- a/python-sdk/nuscenes/lidarseg/lidarseg_utils.py +++ b/python-sdk/nuscenes/lidarseg/lidarseg_utils.py @@ -71,6 +71,9 @@ def plt_to_cv2(points: np.array, coloring: np.array, im, imsize: Tuple[int, int] mat = cv2.cvtColor(mat, cv2.COLOR_RGB2BGR) mat = cv2.resize(mat, imsize) + # Clear off the current figure to prevent an accumulation of figures in memory. + plt.close('all') + return mat diff --git a/python-sdk/tutorials/nuscenes_lidarseg_tutorial.ipynb b/python-sdk/tutorials/nuscenes_lidarseg_tutorial.ipynb index fd9c18e8..d8e1c8c2 100644 --- a/python-sdk/tutorials/nuscenes_lidarseg_tutorial.ipynb +++ b/python-sdk/tutorials/nuscenes_lidarseg_tutorial.ipynb @@ -43,43 +43,12 @@ }, { "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "======\n", - "Loading NuScenes tables for version v1.0-mini...\n", - "Loading nuScenes-lidarseg...\n", - "33 category,\n", - "8 attribute,\n", - "4 visibility,\n", - "911 instance,\n", - "12 sensor,\n", - "120 calibrated_sensor,\n", - "31206 ego_pose,\n", - "8 log,\n", - "10 scene,\n", - "404 sample,\n", - "31206 sample_data,\n", - "18538 sample_annotation,\n", - "4 map,\n", - "404 lidarseg,\n", - "Done loading in 0.4 seconds.\n", - "======\n", - "Reverse indexing ...\n", - "Done reverse indexing in 0.1 seconds.\n", - "======\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "%matplotlib inline\n", "\n", - "import sys, os\n", - "sys.path.insert(0, os.path.expanduser('~/Desktop/nuscenes-devkit/python-sdk'))\n", "from nuscenes import NuScenes\n", "\n", "nusc = NuScenes(version='v1.0-mini', dataroot='/data/sets/nuscenes', verbose=True)" @@ -102,52 +71,9 @@ }, { "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Calculating stats for nuScenes-lidarseg...\n", - " 3 human.pedestrian.wheelchair nbr_points= 0\n", - " 4 human.pedestrian.stroller nbr_points= 0\n", - " 8 animal nbr_points= 0\n", - " 16 vehicle.emergency.ambulance nbr_points= 0\n", - " 17 vehicle.emergency.police nbr_points= 0\n", - " 31 vehicle.ego nbr_points= 0\n", - " 32 vehicle.emergency.firetruck nbr_points= 0\n", - " 22 movable_object.debris nbr_points= 48\n", - " 6 human.pedestrian.police_officer nbr_points= 64\n", - " 2 human.pedestrian.child nbr_points= 230\n", - " 7 human.pedestrian.construction_worker nbr_points= 1,412\n", - " 11 vehicle.bicycle nbr_points= 1,463\n", - " 21 movable_object.pushable_pullable nbr_points= 2,293\n", - " 5 human.pedestrian.personal_mobility nbr_points= 4,096\n", - " 23 static_object.bicycle_rack nbr_points= 4,476\n", - " 20 movable_object.trafficcone nbr_points= 6,206\n", - " 10 vehicle.motorcycle nbr_points= 6,713\n", - " 0 noise nbr_points= 12,561\n", - " 18 vehicle.trailer nbr_points= 12,787\n", - " 13 vehicle.bus.rigid nbr_points= 29,694\n", - " 15 vehicle.construction nbr_points= 39,300\n", - " 12 vehicle.bus.bendy nbr_points= 40,536\n", - " 1 human.pedestrian.adult nbr_points= 43,812\n", - " 19 movable_object.barrier nbr_points= 55,298\n", - " 27 flat.other nbr_points= 150,153\n", - " 14 vehicle.truck nbr_points= 304,234\n", - " 9 vehicle.car nbr_points= 521,237\n", - " 26 flat.terrain nbr_points= 696,526\n", - " 25 flat.sidewalk nbr_points= 746,980\n", - " 29 static.vegetation nbr_points= 1,565,272\n", - " 28 static.manmade nbr_points= 2,067,510\n", - " 30 static.other nbr_points= 3,643,428\n", - " 24 flat.driveable_surface nbr_points= 4,069,879\n", - "Calculated stats for 404 point clouds in 0.1 seconds.\n", - "=====\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "nusc.list_lidarseg_categories(sort_counts=True)" ] @@ -161,52 +87,9 @@ }, { "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{0: 'noise',\n", - " 1: 'human.pedestrian.adult',\n", - " 2: 'human.pedestrian.child',\n", - " 3: 'human.pedestrian.wheelchair',\n", - " 4: 'human.pedestrian.stroller',\n", - " 5: 'human.pedestrian.personal_mobility',\n", - " 6: 'human.pedestrian.police_officer',\n", - " 7: 'human.pedestrian.construction_worker',\n", - " 8: 'animal',\n", - " 9: 'vehicle.car',\n", - " 10: 'vehicle.motorcycle',\n", - " 11: 'vehicle.bicycle',\n", - " 12: 'vehicle.bus.bendy',\n", - " 13: 'vehicle.bus.rigid',\n", - " 14: 'vehicle.truck',\n", - " 15: 'vehicle.construction',\n", - " 16: 'vehicle.emergency.ambulance',\n", - " 17: 'vehicle.emergency.police',\n", - " 18: 'vehicle.trailer',\n", - " 19: 'movable_object.barrier',\n", - " 20: 'movable_object.trafficcone',\n", - " 21: 'movable_object.pushable_pullable',\n", - " 22: 'movable_object.debris',\n", - " 23: 'static_object.bicycle_rack',\n", - " 24: 'flat.driveable_surface',\n", - " 25: 'flat.sidewalk',\n", - " 26: 'flat.terrain',\n", - " 27: 'flat.other',\n", - " 28: 'static.manmade',\n", - " 29: 'static.vegetation',\n", - " 30: 'static.other',\n", - " 31: 'vehicle.ego',\n", - " 32: 'vehicle.emergency.firetruck'}" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "nusc.lidarseg_idx2name_mapping" ] @@ -220,52 +103,9 @@ }, { "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'noise': 0,\n", - " 'human.pedestrian.adult': 1,\n", - " 'human.pedestrian.child': 2,\n", - " 'human.pedestrian.wheelchair': 3,\n", - " 'human.pedestrian.stroller': 4,\n", - " 'human.pedestrian.personal_mobility': 5,\n", - " 'human.pedestrian.police_officer': 6,\n", - " 'human.pedestrian.construction_worker': 7,\n", - " 'animal': 8,\n", - " 'vehicle.car': 9,\n", - " 'vehicle.motorcycle': 10,\n", - " 'vehicle.bicycle': 11,\n", - " 'vehicle.bus.bendy': 12,\n", - " 'vehicle.bus.rigid': 13,\n", - " 'vehicle.truck': 14,\n", - " 'vehicle.construction': 15,\n", - " 'vehicle.emergency.ambulance': 16,\n", - " 'vehicle.emergency.police': 17,\n", - " 'vehicle.trailer': 18,\n", - " 'movable_object.barrier': 19,\n", - " 'movable_object.trafficcone': 20,\n", - " 'movable_object.pushable_pullable': 21,\n", - " 'movable_object.debris': 22,\n", - " 'static_object.bicycle_rack': 23,\n", - " 'flat.driveable_surface': 24,\n", - " 'flat.sidewalk': 25,\n", - " 'flat.terrain': 26,\n", - " 'flat.other': 27,\n", - " 'static.manmade': 28,\n", - " 'static.vegetation': 29,\n", - " 'static.other': 30,\n", - " 'vehicle.ego': 31,\n", - " 'vehicle.emergency.firetruck': 32}" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "nusc.lidarseg_name2idx_mapping" ] @@ -429,7 +269,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We then pass the scene token into ```render_scene_channel_lidarseg``` indicating that we are only interested in construction vehicles and man-made objects.\n", + "We then pass the scene token into ```render_scene_channel_lidarseg``` indicating that we are only interested in construction vehicles and man-made objects. \n", + "\n", + "In addition, you can use `dpi` (to adjust the size of the lidar points) and `imsize` (to adjust the size of the rendered image) to tune the aesthetics of the renderings to your liking.\n", "\n", "(Note: the following code is commented out as it crashes in Jupyter notebooks.)" ] @@ -445,6 +287,7 @@ "# 'CAM_BACK', \n", "# filter_lidarseg_labels=[15, 30],\n", "# verbose=True, \n", + "# dpi=100,\n", "# imsize=(1280, 720))" ] }, @@ -467,6 +310,7 @@ "# 'CAM_BACK',\n", "# filter_lidarseg_labels=[15, 30],\n", "# verbose=True,\n", + "# dpi=100,\n", "# imsize=(1280, 720),\n", "# render_mode='video',\n", "# out_folder=os.path.expanduser('~/Desktop/my_folder'))" @@ -497,6 +341,7 @@ "source": [ "# nusc.render_scene_lidarseg(my_scene['token'], \n", "# filter_lidarseg_labels=[26, 9],\n", + "# verbose=True,\n", "# dpi=100,\n", "# out_path=os.path.expanduser('~/Desktop/my_rendered_scene.avi'))" ] @@ -508,7 +353,14 @@ "## Visualizing LIDAR segmentation predictions\n", "In all the above functions, the labels of the LIDAR pointcloud which have been rendered are the ground truth. If you have trained a model to segment LIDAR pointclouds and have run it on the nuScenes-lidarseg dataset, you can visualize your model's predictions with nuScenes-lidarseg as well!\n", "\n", - "You simply need to pass the path to the .bin file where your predictions for the given sample are to `lidarseg_preds_bin_path` for these functions:\n", + "Each of your .bin files should be a `numpy.uint8` array; as a tip, you can save your predictions as follows:\n", + "```\n", + "np.array(predictions).astype(np.uint8).tofile(bin_file_out)\n", + "```\n", + "- `predictions`: The predictions from your model (e.g. `[30, 5, 18, ..., 30]`)\n", + "- `bin_file_out`: The path to write your .bin file to (e.g. `/some/folder/_lidarseg.bin`)\n", + "\n", + "Then you simply need to pass the path to the .bin file where your predictions for the given sample are to `lidarseg_preds_bin_path` for these functions:\n", "- `list_lidarseg_categories`\n", "- `render_sample_data`\n", "- `render_pointcloud_in_image`\n", From ee848bc92b5c0f0100f5507a3afcfc3f24c84a04 Mon Sep 17 00:00:00 2001 From: whyekit-aptiv Date: Tue, 7 Jul 2020 17:33:18 +0800 Subject: [PATCH 04/17] Include legend for render_sample_data --- python-sdk/nuscenes/nuscenes.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/python-sdk/nuscenes/nuscenes.py b/python-sdk/nuscenes/nuscenes.py index 61448103..7299f751 100644 --- a/python-sdk/nuscenes/nuscenes.py +++ b/python-sdk/nuscenes/nuscenes.py @@ -1229,6 +1229,20 @@ def render_sample_data(self, if filter_lidarseg_labels is not None: coloring = filter_colors(coloring, filter_lidarseg_labels) colors = coloring[points_label] + + show_lidarseg_legend = True + if show_lidarseg_legend: + recs = [] + classes_final = [] + classes = [name for idx, name in sorted(self.nusc.lidarseg_idx2name_mapping.items())] + + for i in range(len(classes)): + if coloring[i][-1] > 0: + recs.append(mpatches.Rectangle((0, 0), 1, 1, fc=coloring[i])) + + # Truncate class names to only first 25 chars so that legend is not excessively long. + classes_final.append(classes[i][:25]) + plt.legend(recs, classes_final, loc='upper center', ncol=3) else: colors = np.minimum(1, dists / axes_limit / np.sqrt(2)) print('Warning: There are no lidarseg labels in {}. Points will be colored according to distance ' @@ -1266,7 +1280,6 @@ def render_sample_data(self, # Limit visible range. ax.set_xlim(-axes_limit, axes_limit) ax.set_ylim(-axes_limit, axes_limit) - elif sensor_modality == 'camera': # Load boxes and image. data_path, boxes, camera_intrinsic = self.nusc.get_sample_data(sample_data_token, From c1fb61ae4c17e98532cb664cbfb941b5a4dfbf78 Mon Sep 17 00:00:00 2001 From: whyekit-aptiv Date: Tue, 7 Jul 2020 17:39:44 +0800 Subject: [PATCH 05/17] Add arg for show_lidarseg_legend in render_sample_data --- python-sdk/nuscenes/nuscenes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-sdk/nuscenes/nuscenes.py b/python-sdk/nuscenes/nuscenes.py index 7299f751..7bb0eb74 100644 --- a/python-sdk/nuscenes/nuscenes.py +++ b/python-sdk/nuscenes/nuscenes.py @@ -515,12 +515,14 @@ def render_sample_data(self, sample_data_token: str, with_anns: bool = True, nsweeps: int = 1, out_path: str = None, underlay_map: bool = True, use_flat_vehicle_coordinates: bool = True, show_lidarseg: bool = False, + show_lidarseg_legend: bool = False, filter_lidarseg_labels: List = None, lidarseg_preds_bin_path: str = None, verbose: bool = True) -> None: self.explorer.render_sample_data(sample_data_token, with_anns, box_vis_level, axes_limit, ax, nsweeps=nsweeps, out_path=out_path, underlay_map=underlay_map, use_flat_vehicle_coordinates=use_flat_vehicle_coordinates, show_lidarseg=show_lidarseg, + show_lidarseg_legend=show_lidarseg_legend, filter_lidarseg_labels=filter_lidarseg_labels, lidarseg_preds_bin_path=lidarseg_preds_bin_path, verbose=verbose) @@ -1098,6 +1100,7 @@ def render_sample_data(self, underlay_map: bool = True, use_flat_vehicle_coordinates: bool = True, show_lidarseg: bool = False, + show_lidarseg_legend: bool = False, filter_lidarseg_labels: List = None, lidarseg_preds_bin_path: str = None, verbose: bool = True) -> None: @@ -1117,6 +1120,7 @@ def render_sample_data(self, setting is more correct and rotates the plot by ~90 degrees. :param show_lidarseg: When set to True, the lidar data is colored with the segmentation labels. When set to False, the colors of the lidar data represent the distance from the center of the ego vehicle. + :param show_lidarseg_legend: Whether to display the legend for the lidarseg labels in the frame. :param filter_lidarseg_labels: Only show lidar points which belong to the given list of classes. If None or the list is empty, all classes will be displayed. :param lidarseg_preds_bin_path: A path to the .bin file which contains the user's lidar segmentation @@ -1230,7 +1234,6 @@ def render_sample_data(self, coloring = filter_colors(coloring, filter_lidarseg_labels) colors = coloring[points_label] - show_lidarseg_legend = True if show_lidarseg_legend: recs = [] classes_final = [] From c5c74447507ecfcad17fa6e678194d70f3225b3e Mon Sep 17 00:00:00 2001 From: whyekit-aptiv Date: Tue, 7 Jul 2020 17:55:43 +0800 Subject: [PATCH 06/17] Adjust aesthetics of legend in render_sample_data --- python-sdk/nuscenes/nuscenes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python-sdk/nuscenes/nuscenes.py b/python-sdk/nuscenes/nuscenes.py index 7bb0eb74..34816c13 100644 --- a/python-sdk/nuscenes/nuscenes.py +++ b/python-sdk/nuscenes/nuscenes.py @@ -1245,7 +1245,8 @@ def render_sample_data(self, # Truncate class names to only first 25 chars so that legend is not excessively long. classes_final.append(classes[i][:25]) - plt.legend(recs, classes_final, loc='upper center', ncol=3) + plt.legend(recs, classes_final, loc='upper left', ncol=1, + bbox_to_anchor=(1.05, 1.0)) else: colors = np.minimum(1, dists / axes_limit / np.sqrt(2)) print('Warning: There are no lidarseg labels in {}. Points will be colored according to distance ' From 40873e42a99f8a3bf0b461d4f895a8ad2c341107 Mon Sep 17 00:00:00 2001 From: whyekit-aptiv Date: Tue, 7 Jul 2020 18:00:27 +0800 Subject: [PATCH 07/17] Update colormap --- python-sdk/nuscenes/utils/color_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-sdk/nuscenes/utils/color_map.py b/python-sdk/nuscenes/utils/color_map.py index a08a883f..d9b1290f 100644 --- a/python-sdk/nuscenes/utils/color_map.py +++ b/python-sdk/nuscenes/utils/color_map.py @@ -32,7 +32,7 @@ def get_colormap() -> Dict[str, Iterable[int]]: "movable_object.pushable_pullable": [105, 105, 105], # Dimgrey "movable_object.debris": [210, 105, 30], # Chocolate "static_object.bicycle_rack": [188, 143, 143], # Rosybrown - "flat.driveable_surface": [128, 64, 128], # Dark purple + "flat.driveable_surface": [0, 207, 191], # nuTonomy green "flat.sidewalk": [75, 0, 75], "flat.terrain": [112, 180, 60], "flat.other": [175, 0, 75], From 5a4d331645a2b341098f3f4d339435ad87b22035 Mon Sep 17 00:00:00 2001 From: whyekit-aptiv Date: Tue, 7 Jul 2020 19:09:05 +0800 Subject: [PATCH 08/17] Make creating legend modular --- .../nuscenes/lidarseg/lidarseg_utils.py | 19 +++++++++- python-sdk/nuscenes/nuscenes.py | 36 +++++-------------- 2 files changed, 26 insertions(+), 29 deletions(-) diff --git a/python-sdk/nuscenes/lidarseg/lidarseg_utils.py b/python-sdk/nuscenes/lidarseg/lidarseg_utils.py index c063df38..dad7574e 100644 --- a/python-sdk/nuscenes/lidarseg/lidarseg_utils.py +++ b/python-sdk/nuscenes/lidarseg/lidarseg_utils.py @@ -1,7 +1,6 @@ # nuScenes dev-kit. # Code written by Fong Whye Kit, 2020. -import colorsys from typing import Dict, Iterable, List, Tuple import cv2 @@ -152,3 +151,21 @@ def _array_in_list(arr: List, list_arrays: List) -> bool: filter_lidarseg_labels.append(i) return filter_lidarseg_labels + + +def create_lidarseg_legend(labels_to_include_in_legend, idx2name, name2color, + loc: str = 'upper center', ncol: int = 3, bbox_to_anchor: Tuple = None): + import matplotlib.patches as mpatches + recs = [] + classes_final = [] + classes = [name for idx, name in sorted(idx2name.items())] + + for i in range(len(classes)): + if labels_to_include_in_legend is None or i in labels_to_include_in_legend: + name = classes[i] + recs.append(mpatches.Rectangle((0, 0), 1, 1, fc=np.array(name2color[name]) / 255)) + + # Truncate class names to only first 25 chars so that legend is not excessively long. + classes_final.append(classes[i][:25]) + + plt.legend(recs, classes_final, loc=loc, ncol=ncol, bbox_to_anchor=bbox_to_anchor) diff --git a/python-sdk/nuscenes/nuscenes.py b/python-sdk/nuscenes/nuscenes.py index 34816c13..440b32f0 100644 --- a/python-sdk/nuscenes/nuscenes.py +++ b/python-sdk/nuscenes/nuscenes.py @@ -22,7 +22,7 @@ from tqdm import tqdm from nuscenes.lidarseg.lidarseg_utils import filter_colors, colormap_to_colors, plt_to_cv2, get_stats, \ - get_key_from_value, get_labels_in_coloring + get_key_from_value, get_labels_in_coloring, create_lidarseg_legend from nuscenes.utils.data_classes import LidarPointCloud, RadarPointCloud, Box from nuscenes.utils.geometry_utils import view_points, box_in_image, BoxVisibility, transform_matrix from nuscenes.utils.map_mask import MapMask @@ -909,13 +909,8 @@ def render_pointcloud_in_image(self, # Produce a legend with the unique colors from the scatter. if pointsensor_channel == 'LIDAR_TOP' and show_lidarseg and show_lidarseg_legend: - recs = [] - classes_final = [] - classes = [name for idx, name in sorted(self.nusc.lidarseg_idx2name_mapping.items())] - - # A scatter plot is used for displaying the lidarseg points; however, the scatter plot takes in colors - # as an array of RGB values, and thus the colormap needs to be converted to the appropriate format for - # later use. + # Since the labels are stored as class indices, we get the RGB colors from the colormap in an array where + # the position of the RGB color corresponds to the index of the class it represents. color_legend = colormap_to_colors(self.nusc.colormap, self.nusc.lidarseg_name2idx_mapping) # If user does not specify a filter, then set the filter to contain the classes present in the pointcloud @@ -924,14 +919,8 @@ def render_pointcloud_in_image(self, if filter_lidarseg_labels is None: filter_lidarseg_labels = get_labels_in_coloring(color_legend, coloring) - for i in range(len(classes)): - # Create legend only for labels specified in the lidarseg filter. - if filter_lidarseg_labels is None or i in filter_lidarseg_labels: - recs.append(mpatches.Rectangle((0, 0), 1, 1, fc=color_legend[i])) - - # Truncate class names to only first 25 chars so that legend is not excessively long. - classes_final.append(classes[i][:25]) - plt.legend(recs, classes_final, loc='upper center', ncol=3) + create_lidarseg_legend(filter_lidarseg_labels, + self.nusc.lidarseg_idx2name_mapping, self.nusc.colormap) if out_path is not None: plt.savefig(out_path, bbox_inches='tight', pad_inches=0, dpi=200) @@ -1235,18 +1224,9 @@ def render_sample_data(self, colors = coloring[points_label] if show_lidarseg_legend: - recs = [] - classes_final = [] - classes = [name for idx, name in sorted(self.nusc.lidarseg_idx2name_mapping.items())] - - for i in range(len(classes)): - if coloring[i][-1] > 0: - recs.append(mpatches.Rectangle((0, 0), 1, 1, fc=coloring[i])) - - # Truncate class names to only first 25 chars so that legend is not excessively long. - classes_final.append(classes[i][:25]) - plt.legend(recs, classes_final, loc='upper left', ncol=1, - bbox_to_anchor=(1.05, 1.0)) + create_lidarseg_legend(filter_lidarseg_labels, + self.nusc.lidarseg_idx2name_mapping, self.nusc.colormap, + loc='upper left', ncol=1, bbox_to_anchor=(1.05, 1.0)) else: colors = np.minimum(1, dists / axes_limit / np.sqrt(2)) print('Warning: There are no lidarseg labels in {}. Points will be colored according to distance ' From a274686511f1fbc549c8d4337d6b06338399f16b Mon Sep 17 00:00:00 2001 From: whyekit-aptiv Date: Tue, 7 Jul 2020 19:19:05 +0800 Subject: [PATCH 09/17] Ensure only labels in pc is included in legend --- python-sdk/nuscenes/nuscenes.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python-sdk/nuscenes/nuscenes.py b/python-sdk/nuscenes/nuscenes.py index 440b32f0..795c0e7e 100644 --- a/python-sdk/nuscenes/nuscenes.py +++ b/python-sdk/nuscenes/nuscenes.py @@ -1224,6 +1224,12 @@ def render_sample_data(self, colors = coloring[points_label] if show_lidarseg_legend: + # If user does not specify a filter, then set the filter to contain the classes present in + # the pointcloud after it has been projected onto the image; this will allow displaying the + # legend only for classes which are present in the image (instead of all the classes). + if filter_lidarseg_labels is None: + filter_lidarseg_labels = get_labels_in_coloring(coloring, colors) + create_lidarseg_legend(filter_lidarseg_labels, self.nusc.lidarseg_idx2name_mapping, self.nusc.colormap, loc='upper left', ncol=1, bbox_to_anchor=(1.05, 1.0)) From 7b3b50faf7e9a8fb72834199028778341ba73053 Mon Sep 17 00:00:00 2001 From: whyekit-aptiv Date: Tue, 7 Jul 2020 19:29:28 +0800 Subject: [PATCH 10/17] Explain verbose --- .../nuscenes_lidarseg_tutorial.ipynb | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/python-sdk/tutorials/nuscenes_lidarseg_tutorial.ipynb b/python-sdk/tutorials/nuscenes_lidarseg_tutorial.ipynb index d8e1c8c2..3f6347bd 100644 --- a/python-sdk/tutorials/nuscenes_lidarseg_tutorial.ipynb +++ b/python-sdk/tutorials/nuscenes_lidarseg_tutorial.ipynb @@ -49,6 +49,8 @@ "source": [ "%matplotlib inline\n", "\n", + "import sys, os\n", + "sys.path.insert(0, os.path.expanduser('~/Desktop/nuscenes-devkit/python-sdk'))\n", "from nuscenes import NuScenes\n", "\n", "nusc = NuScenes(version='v1.0-mini', dataroot='/data/sets/nuscenes', verbose=True)" @@ -197,6 +199,25 @@ "Now only points in the pointcloud belonging to trucks and trailers are filtered out for your viewing pleasure. " ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In addition, you can display a legend which indicates the color for each class by using `show_lidarseg_legend`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nusc.render_sample_data(sample_data_token,\n", + " with_anns=False,\n", + " show_lidarseg=True,\n", + " show_lidarseg_legend=True)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -217,8 +238,7 @@ " render_intensity=False,\n", " show_lidarseg=True,\n", " filter_lidarseg_labels=[9, 14, 18],\n", - " show_lidarseg_legend=True,\n", - " verbose=True)" + " show_lidarseg_legend=True)" ] }, { @@ -237,8 +257,7 @@ "source": [ "nusc.render_sample(my_sample['token'],\n", " show_lidarseg=True,\n", - " filter_lidarseg_labels=[9, 14, 18],\n", - " verbose=True)" + " filter_lidarseg_labels=[9, 14, 18])" ] }, { @@ -269,7 +288,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We then pass the scene token into ```render_scene_channel_lidarseg``` indicating that we are only interested in construction vehicles and man-made objects. \n", + "We then pass the scene token into ```render_scene_channel_lidarseg``` indicating that we are only interested in construction vehicles and man-made objects (here, we set `verbose=True` to produce a window which will allows us to see the frames as they are being random). \n", "\n", "In addition, you can use `dpi` (to adjust the size of the lidar points) and `imsize` (to adjust the size of the rendered image) to tune the aesthetics of the renderings to your liking.\n", "\n", @@ -388,7 +407,6 @@ " show_lidarseg=True,\n", " filter_lidarseg_labels=[9, 14, 18],\n", " show_lidarseg_legend=True,\n", - " verbose=True,\n", " lidarseg_preds_bin_path=my_predictions_bin_file)" ] }, From 11d7af5001e2fc62107825e480c8fd877e9b3611 Mon Sep 17 00:00:00 2001 From: whyekit-aptiv Date: Tue, 7 Jul 2020 21:47:04 +0800 Subject: [PATCH 11/17] Check that filter_lidarseg_labels is either a list or np.array --- python-sdk/nuscenes/nuscenes.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/python-sdk/nuscenes/nuscenes.py b/python-sdk/nuscenes/nuscenes.py index 795c0e7e..f99936d3 100644 --- a/python-sdk/nuscenes/nuscenes.py +++ b/python-sdk/nuscenes/nuscenes.py @@ -465,7 +465,7 @@ def get_sample_lidarseg_stats(self, sample_token: str, sort_counts: bool = True, for class_name, count in out: if count > 0: idx = get_key_from_value(self.lidarseg_idx2name_mapping, class_name) - print('{:3} {:35} n={:12,}'.format(idx, class_name, count)) + print('{:3} {:40} n={:12,}'.format(idx, class_name, count)) print('=' * len(header)) @@ -825,6 +825,11 @@ def map_pointcloud_to_image(self, colors = colormap_to_colors(self.nusc.colormap, self.nusc.lidarseg_name2idx_mapping) if filter_lidarseg_labels is not None: + # Ensure that filter_lidarseg_labels is an iterable. + assert isinstance(filter_lidarseg_labels, (list, np.ndarray)), \ + 'Error: filter_lidarseg_labels should be a list of class indices, eg. [9], [10, 21].' + + # Filter to get only the colors of the desired classes. colors = filter_colors(colors, filter_lidarseg_labels) coloring = colors[points_label] else: @@ -1220,6 +1225,11 @@ def render_sample_data(self, coloring = colormap_to_colors(self.nusc.colormap, self.nusc.lidarseg_name2idx_mapping) if filter_lidarseg_labels is not None: + # Ensure that filter_lidarseg_labels is an iterable. + assert isinstance(filter_lidarseg_labels, (list, np.ndarray)), \ + 'Error: filter_lidarseg_labels should be a list of class indices, eg. [9], [10, 21].' + + # Filter to get only the colors of the desired classes. coloring = filter_colors(coloring, filter_lidarseg_labels) colors = coloring[points_label] From 01d75841be99024a8041f3e76da07f38a71cc7c7 Mon Sep 17 00:00:00 2001 From: whyekit-aptiv Date: Tue, 7 Jul 2020 22:04:19 +0800 Subject: [PATCH 12/17] Allow user to list stats by class index --- python-sdk/nuscenes/nuscenes.py | 34 +++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/python-sdk/nuscenes/nuscenes.py b/python-sdk/nuscenes/nuscenes.py index f99936d3..d3083772 100644 --- a/python-sdk/nuscenes/nuscenes.py +++ b/python-sdk/nuscenes/nuscenes.py @@ -409,19 +409,22 @@ def box_velocity(self, sample_annotation_token: str, max_time_diff: float = 1.5) else: return pos_diff / time_diff - def get_sample_lidarseg_stats(self, sample_token: str, sort_counts: bool = True, + def get_sample_lidarseg_stats(self, sample_token: str, sort_by: str = 'count', lidarseg_preds_bin_path: str = None) -> None: """ Print the number of points for each class in the lidar pointcloud of a sample. Classes with have no points in the pointcloud will not be printed. :param sample_token: Sample token. - :param sort_counts: If True, the stats will be printed in ascending order of frequency; if False, - the stats will be printed alphabetically according to class name. + :param sort_by: One of three options: count / name / index. If 'count`, the stats will be printed in + ascending order of frequency; if `name`, the stats will be printed alphabetically + according to class name; if `index`, the stats will be printed in ascending order of index. :param lidarseg_preds_bin_path: A path to the .bin file which contains the user's lidar segmentation predictions for the sample. """ assert hasattr(self, 'lidarseg'), 'Error: You have no lidarseg data; unable to get ' \ 'statistics for segmentation of the point cloud.' + assert sort_by in ['count', 'name', 'index'], 'Error: sort_by can only be one of the following: ' \ + 'count / name / index.' sample_rec = self.get('sample', sample_token) ref_sd_token = sample_rec['data']['LIDAR_TOP'] @@ -457,10 +460,12 @@ def get_sample_lidarseg_stats(self, sample_token: str, sort_counts: bool = True, for i in range(len(lidarseg_counts)): lidarseg_counts_dict[self.lidarseg_idx2name_mapping[i]] = lidarseg_counts[i] - if sort_counts: + if sort_by == 'count': out = sorted(lidarseg_counts_dict.items(), key=lambda item: item[1]) - else: + elif sort_by == 'name': out = sorted(lidarseg_counts_dict.items()) + else: + out = lidarseg_counts_dict.items() for class_name, count in out: if count > 0: @@ -472,8 +477,8 @@ def get_sample_lidarseg_stats(self, sample_token: str, sort_counts: bool = True, def list_categories(self) -> None: self.explorer.list_categories() - def list_lidarseg_categories(self, sort_counts: bool = True) -> None: - self.explorer.list_lidarseg_categories(sort_counts=sort_counts) + def list_lidarseg_categories(self, sort_by: str = 'count') -> None: + self.explorer.list_lidarseg_categories(sort_by=sort_by) def list_attributes(self) -> None: self.explorer.list_attributes() @@ -626,14 +631,17 @@ def list_categories(self) -> None: np.mean(stats[:, 2]), np.std(stats[:, 2]), np.mean(stats[:, 3]), np.std(stats[:, 3]))) - def list_lidarseg_categories(self, sort_counts: bool = True) -> None: + def list_lidarseg_categories(self, sort_by: str = 'count') -> None: """ Print categories and counts of the lidarseg data. These stats only cover the split specified in nusc.version. - :param sort_counts: If True, the stats will be printed in ascending order of frequency; if False, - the stats will be printed alphabetically according to class name. + :param sort_by: One of three options: count / name / index. If 'count`, the stats will be printed in + ascending order of frequency; if `name`, the stats will be printed alphabetically + according to class name; if `index`, the stats will be printed in ascending order of index. """ assert hasattr(self.nusc, 'lidarseg'), 'Error: nuScenes-lidarseg not installed!' + assert sort_by in ['count', 'name', 'index'], 'Error: sort_by can only be one of the following: ' \ + 'count / name / index.' print('Calculating stats for nuScenes-lidarseg...') start_time = time.time() @@ -654,10 +662,12 @@ def list_lidarseg_categories(self, sort_counts: bool = True) -> None: for i in range(len(lidarseg_counts)): lidarseg_counts_dict[self.nusc.lidarseg_idx2name_mapping[i]] = lidarseg_counts[i] - if sort_counts: + if sort_by == 'count': out = sorted(lidarseg_counts_dict.items(), key=lambda item: item[1]) - else: + elif sort_by == 'name': out = sorted(lidarseg_counts_dict.items()) + else: + out = lidarseg_counts_dict.items() # Print frequency counts of each class in the lidarseg dataset. for class_name, count in out: From 2d0c84a76f77f9eda32baa5a39ad6f93b754e7ad Mon Sep 17 00:00:00 2001 From: whyekit-aptiv Date: Tue, 7 Jul 2020 22:15:42 +0800 Subject: [PATCH 13/17] Update docstrings --- python-sdk/nuscenes/lidarseg/lidarseg_utils.py | 15 ++++++++++++++- python-sdk/nuscenes/nuscenes.py | 7 ++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/python-sdk/nuscenes/lidarseg/lidarseg_utils.py b/python-sdk/nuscenes/lidarseg/lidarseg_utils.py index dad7574e..30f9d45c 100644 --- a/python-sdk/nuscenes/lidarseg/lidarseg_utils.py +++ b/python-sdk/nuscenes/lidarseg/lidarseg_utils.py @@ -4,6 +4,7 @@ from typing import Dict, Iterable, List, Tuple import cv2 +import matplotlib.patches as mpatches import matplotlib.pyplot as plt import numpy as np from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas @@ -155,7 +156,19 @@ def _array_in_list(arr: List, list_arrays: List) -> bool: def create_lidarseg_legend(labels_to_include_in_legend, idx2name, name2color, loc: str = 'upper center', ncol: int = 3, bbox_to_anchor: Tuple = None): - import matplotlib.patches as mpatches + """ + Given a list of class indices, the mapping from class index to class name, and the mapping from class name + to class color, produce a legend which shows the color and the corresponding class name. + :param labels_to_include_in_legend: + :param idx2name: The mapping from class index to class name. + :param name2color: The mapping from class name to class color. + :param loc: The location of the legend. + :param ncol: The number of columns that the legend has. + :param bbox_to_anchor: A 2-tuple (x, y) which places the top-left corner of the legend specified by loc + at x, y. The origin is at the bottom-left corner and x and y are normalized between + 0 and 1 (i.e. x > 1 and / or y > 1 will place the legend outside the plot. + """ + recs = [] classes_final = [] classes = [name for idx, name in sorted(idx2name.items())] diff --git a/python-sdk/nuscenes/nuscenes.py b/python-sdk/nuscenes/nuscenes.py index d3083772..e2cf219c 100644 --- a/python-sdk/nuscenes/nuscenes.py +++ b/python-sdk/nuscenes/nuscenes.py @@ -12,7 +12,6 @@ import cv2 import matplotlib.pyplot as plt -import matplotlib.patches as mpatches import numpy as np import sklearn.metrics from PIL import Image @@ -417,7 +416,8 @@ def get_sample_lidarseg_stats(self, sample_token: str, sort_by: str = 'count', :param sample_token: Sample token. :param sort_by: One of three options: count / name / index. If 'count`, the stats will be printed in ascending order of frequency; if `name`, the stats will be printed alphabetically - according to class name; if `index`, the stats will be printed in ascending order of index. + according to class name; if `index`, the stats will be printed in ascending order of + class index. :param lidarseg_preds_bin_path: A path to the .bin file which contains the user's lidar segmentation predictions for the sample. """ @@ -637,7 +637,8 @@ def list_lidarseg_categories(self, sort_by: str = 'count') -> None: the split specified in nusc.version. :param sort_by: One of three options: count / name / index. If 'count`, the stats will be printed in ascending order of frequency; if `name`, the stats will be printed alphabetically - according to class name; if `index`, the stats will be printed in ascending order of index. + according to class name; if `index`, the stats will be printed in ascending order of + class index. """ assert hasattr(self.nusc, 'lidarseg'), 'Error: nuScenes-lidarseg not installed!' assert sort_by in ['count', 'name', 'index'], 'Error: sort_by can only be one of the following: ' \ From e5f904161dff598c8b554a6b4487acaa8a10480f Mon Sep 17 00:00:00 2001 From: whyekit-aptiv Date: Tue, 7 Jul 2020 22:30:16 +0800 Subject: [PATCH 14/17] Update tutorial to demo sort_by --- .../nuscenes_lidarseg_tutorial.ipynb | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/python-sdk/tutorials/nuscenes_lidarseg_tutorial.ipynb b/python-sdk/tutorials/nuscenes_lidarseg_tutorial.ipynb index 3f6347bd..d4258881 100644 --- a/python-sdk/tutorials/nuscenes_lidarseg_tutorial.ipynb +++ b/python-sdk/tutorials/nuscenes_lidarseg_tutorial.ipynb @@ -49,8 +49,6 @@ "source": [ "%matplotlib inline\n", "\n", - "import sys, os\n", - "sys.path.insert(0, os.path.expanduser('~/Desktop/nuscenes-devkit/python-sdk'))\n", "from nuscenes import NuScenes\n", "\n", "nusc = NuScenes(version='v1.0-mini', dataroot='/data/sets/nuscenes', verbose=True)" @@ -68,7 +66,7 @@ "metadata": {}, "source": [ "## Statistics of lidarseg dataset for the v1.0-mini split\n", - "Let's get a quick feel of the lidarseg dataset by looking at what classes are in it and the number of points belonging to each class. The classes will be sorted in ascending order based on the number of points (since `sort_counts=True` below; if you wish to sort the statistics by alphabetical order, just set `sort_counts=False`)." + "Let's get a quick feel of the lidarseg dataset by looking at what classes are in it and the number of points belonging to each class. The classes will be sorted in ascending order based on the number of points (since `sort_by='count'` below); you can also sort the classes by class name or class index by setting `sort_by='name'` or `sort_by='index'` respectively." ] }, { @@ -77,7 +75,7 @@ "metadata": {}, "outputs": [], "source": [ - "nusc.list_lidarseg_categories(sort_counts=True)" + "nusc.list_lidarseg_categories(sort_by='count')" ] }, { @@ -143,14 +141,14 @@ "metadata": {}, "outputs": [], "source": [ - "nusc.get_sample_lidarseg_stats(my_sample['token'], sort_counts=True)" + "nusc.get_sample_lidarseg_stats(my_sample['token'], sort_by='count')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "By doing ```sort_counts=True```, the classes and their respective frequency counts are printed in ascending order; On the other hand, ```sort_counts=False``` will print the classes and their respective frequency counts in alphabetical order." + "By doing `sort_by='count'`, the classes and their respective frequency counts are printed in ascending order; you can also do `sort_by='name'` and `sort_by='index'` here as well." ] }, { @@ -304,7 +302,7 @@ "# import os\n", "# nusc.render_scene_channel_lidarseg(my_scene['token'], \n", "# 'CAM_BACK', \n", - "# filter_lidarseg_labels=[15, 30],\n", + "# filter_lidarseg_labels=[15, 24],\n", "# verbose=True, \n", "# dpi=100,\n", "# imsize=(1280, 720))" @@ -327,7 +325,7 @@ "source": [ "# nusc.render_scene_channel_lidarseg(my_scene['token'],\n", "# 'CAM_BACK',\n", - "# filter_lidarseg_labels=[15, 30],\n", + "# filter_lidarseg_labels=[15, 24],\n", "# verbose=True,\n", "# dpi=100,\n", "# imsize=(1280, 720),\n", @@ -355,7 +353,9 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ "# nusc.render_scene_lidarseg(my_scene['token'], \n", @@ -434,7 +434,7 @@ "\n", "# nusc.render_scene_channel_lidarseg(my_scene['token'], \n", "# 'CAM_BACK', \n", - "# filter_lidarseg_labels=[15, 30],\n", + "# filter_lidarseg_labels=[15, 24],\n", "# verbose=True, \n", "# imsize=(1280, 720),\n", "# lidarseg_preds_folder=my_folder_of_predictions)" From 5d01b17863144c268f5a1b58cfdb72ae8b7931ed Mon Sep 17 00:00:00 2001 From: whyekit-aptiv Date: Tue, 7 Jul 2020 23:06:23 +0800 Subject: [PATCH 15/17] Add unit tests --- python-sdk/nuscenes/tests/test_lidarseg.py | 41 ++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 python-sdk/nuscenes/tests/test_lidarseg.py diff --git a/python-sdk/nuscenes/tests/test_lidarseg.py b/python-sdk/nuscenes/tests/test_lidarseg.py new file mode 100644 index 00000000..8b636d7f --- /dev/null +++ b/python-sdk/nuscenes/tests/test_lidarseg.py @@ -0,0 +1,41 @@ +import unittest +import os + +from nuscenes import NuScenes + + +class TestNuScenesLidarseg(unittest.TestCase): + def setUp(self): + assert 'NUSCENES' in os.environ, 'Set NUSCENES env. variable to enable tests.' + self.nusc = NuScenes(version='v1.0-mini', dataroot=os.environ['NUSCENES'], verbose=False) + + def test_num_classes(self) -> None: + """ + Check that the correct number of classes (32 classes) are loaded. + """ + self.assertEqual(len(self.nusc.lidarseg_idx2name_mapping), 32) + + def test_num_colors(self) -> None: + """ + Check that the number of colors in the colormap matches the number of classes. + """ + num_classes = len(self.nusc.lidarseg_idx2name_mapping) + num_colors = len(self.nusc.colormap) + self.assertEqual(num_colors, num_classes) + + def test_classes(self) -> None: + """ + Check that the class names match the ones in the colormap, and are in the same order. + """ + classes_in_colormap = list(self.nusc.colormap.keys()) + for name, idx in self.nusc.lidarseg_name2idx_mapping.items(): + self.assertEqual(name, classes_in_colormap[idx]) + + +if __name__ == '__main__': + # Runs the tests without throwing errors. + test = TestNuScenesLidarseg() + test.setUp() + test.test_num_classes() + test.test_num_colors() + test.test_classes() From cdc6fbe08d3d911f77e2adebb96c818e81b89109 Mon Sep 17 00:00:00 2001 From: whyekit-aptiv Date: Wed, 8 Jul 2020 17:08:51 +0800 Subject: [PATCH 16/17] Make painting of labels modular --- .../nuscenes/lidarseg/lidarseg_utils.py | 34 ++++++++++++++ python-sdk/nuscenes/nuscenes.py | 45 ++++++------------- 2 files changed, 47 insertions(+), 32 deletions(-) diff --git a/python-sdk/nuscenes/lidarseg/lidarseg_utils.py b/python-sdk/nuscenes/lidarseg/lidarseg_utils.py index 30f9d45c..1de59b31 100644 --- a/python-sdk/nuscenes/lidarseg/lidarseg_utils.py +++ b/python-sdk/nuscenes/lidarseg/lidarseg_utils.py @@ -182,3 +182,37 @@ def create_lidarseg_legend(labels_to_include_in_legend, idx2name, name2color, classes_final.append(classes[i][:25]) plt.legend(recs, classes_final, loc=loc, ncol=ncol, bbox_to_anchor=bbox_to_anchor) + + +def paint_points_label(lidarseg_labels_filename: str, filter_lidarseg_labels: List[int], + name2idx: Dict[str, int], colormap: Dict[str, List[int]]) -> np.ndarray: + """ + + """ + + # Load labels from .bin file. + points_label = np.fromfile(lidarseg_labels_filename, dtype=np.uint8) # [num_points] + + # Given a colormap (class name -> RGB color) and a mapping from class name to class index, + # get an array of RGB values where each color sits at the index in the array corresponding + # to the class index. + colors = colormap_to_colors(colormap, name2idx) # Shape: [num_class, 3] + + if filter_lidarseg_labels is not None: + # Ensure that filter_lidarseg_labels is an iterable. + assert isinstance(filter_lidarseg_labels, (list, np.ndarray)), \ + 'Error: filter_lidarseg_labels should be a list of class indices, eg. [9], [10, 21].' + + # Check that class indices in filter_lidarseg_labels are valid. + assert all([0 <= x < len(name2idx) for x in filter_lidarseg_labels]), \ + 'All class indices in filter_lidarseg_labels should ' \ + 'be between 0 and {}'.format(len(name2idx) - 1) + + # Filter to get only the colors of the desired classes; this is done by setting the + # alpha channel of the classes to be viewed to 1, and the rest to 0. + colors = filter_colors(colors, filter_lidarseg_labels) # Shape: [num_class, 4] + + # Paint each label with its respective RGBA value. + coloring = colors[points_label] # Shape: [num_points, 4] + + return coloring diff --git a/python-sdk/nuscenes/nuscenes.py b/python-sdk/nuscenes/nuscenes.py index e2cf219c..6805beaa 100644 --- a/python-sdk/nuscenes/nuscenes.py +++ b/python-sdk/nuscenes/nuscenes.py @@ -21,7 +21,7 @@ from tqdm import tqdm from nuscenes.lidarseg.lidarseg_utils import filter_colors, colormap_to_colors, plt_to_cv2, get_stats, \ - get_key_from_value, get_labels_in_coloring, create_lidarseg_legend + get_key_from_value, get_labels_in_coloring, create_lidarseg_legend, paint_points_label from nuscenes.utils.data_classes import LidarPointCloud, RadarPointCloud, Box from nuscenes.utils.geometry_utils import view_points, box_in_image, BoxVisibility, transform_matrix from nuscenes.utils.map_mask import MapMask @@ -828,21 +828,9 @@ def map_pointcloud_to_image(self, lidarseg_labels_filename = None if lidarseg_labels_filename: - points_label = np.fromfile(lidarseg_labels_filename, dtype=np.uint8) - - # A scatter plot is used for displaying the lidarseg points; however, the scatter plot takes in colors - # as an array of RGB values, and thus the colormap needs to be converted to the appropriate format for - # later use. - colors = colormap_to_colors(self.nusc.colormap, self.nusc.lidarseg_name2idx_mapping) - - if filter_lidarseg_labels is not None: - # Ensure that filter_lidarseg_labels is an iterable. - assert isinstance(filter_lidarseg_labels, (list, np.ndarray)), \ - 'Error: filter_lidarseg_labels should be a list of class indices, eg. [9], [10, 21].' - - # Filter to get only the colors of the desired classes. - colors = filter_colors(colors, filter_lidarseg_labels) - coloring = colors[points_label] + # Paint each label in the pointcloud with a RGBA value. + coloring = paint_points_label(lidarseg_labels_filename, filter_lidarseg_labels, + self.nusc.lidarseg_name2idx_mapping, self.nusc.colormap) else: coloring = depths print('Warning: There are no lidarseg labels in {}. Points will be colored according to distance ' @@ -1228,28 +1216,21 @@ def render_sample_data(self, lidarseg_labels_filename = None if lidarseg_labels_filename: - points_label = np.fromfile(lidarseg_labels_filename, dtype=np.uint8) - - # A scatter plot is used for displaying the lidarseg points; however, the scatter plot takes - # in colors as an array of RGB values, and thus the colormap needs to be converted to the - # appropriate format. - coloring = colormap_to_colors(self.nusc.colormap, self.nusc.lidarseg_name2idx_mapping) - - if filter_lidarseg_labels is not None: - # Ensure that filter_lidarseg_labels is an iterable. - assert isinstance(filter_lidarseg_labels, (list, np.ndarray)), \ - 'Error: filter_lidarseg_labels should be a list of class indices, eg. [9], [10, 21].' - - # Filter to get only the colors of the desired classes. - coloring = filter_colors(coloring, filter_lidarseg_labels) - colors = coloring[points_label] + # Paint each label in the pointcloud with a RGBA value. + colors = paint_points_label(lidarseg_labels_filename, filter_lidarseg_labels, + self.nusc.lidarseg_name2idx_mapping, self.nusc.colormap) if show_lidarseg_legend: + # Since the labels are stored as class indices, we get the RGB colors from the colormap + # in an array where the position of the RGB color corresponds to the index of the class + # it represents. + color_legend = colormap_to_colors(self.nusc.colormap, self.nusc.lidarseg_name2idx_mapping) + # If user does not specify a filter, then set the filter to contain the classes present in # the pointcloud after it has been projected onto the image; this will allow displaying the # legend only for classes which are present in the image (instead of all the classes). if filter_lidarseg_labels is None: - filter_lidarseg_labels = get_labels_in_coloring(coloring, colors) + filter_lidarseg_labels = get_labels_in_coloring(color_legend, colors) create_lidarseg_legend(filter_lidarseg_labels, self.nusc.lidarseg_idx2name_mapping, self.nusc.colormap, From 4abb2daf862de71987fa2bbfe6ab5943815ff93a Mon Sep 17 00:00:00 2001 From: whyekit-aptiv Date: Wed, 8 Jul 2020 17:37:13 +0800 Subject: [PATCH 17/17] Add docstrings --- python-sdk/nuscenes/lidarseg/lidarseg_utils.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/python-sdk/nuscenes/lidarseg/lidarseg_utils.py b/python-sdk/nuscenes/lidarseg/lidarseg_utils.py index 1de59b31..caee229f 100644 --- a/python-sdk/nuscenes/lidarseg/lidarseg_utils.py +++ b/python-sdk/nuscenes/lidarseg/lidarseg_utils.py @@ -154,12 +154,13 @@ def _array_in_list(arr: List, list_arrays: List) -> bool: return filter_lidarseg_labels -def create_lidarseg_legend(labels_to_include_in_legend, idx2name, name2color, +def create_lidarseg_legend(labels_to_include_in_legend: List[int], + idx2name: Dict[int, str], name2color: Dict[str, List[int]], loc: str = 'upper center', ncol: int = 3, bbox_to_anchor: Tuple = None): """ Given a list of class indices, the mapping from class index to class name, and the mapping from class name to class color, produce a legend which shows the color and the corresponding class name. - :param labels_to_include_in_legend: + :param labels_to_include_in_legend: Labels to show in the legend. :param idx2name: The mapping from class index to class name. :param name2color: The mapping from class name to class color. :param loc: The location of the legend. @@ -187,7 +188,16 @@ def create_lidarseg_legend(labels_to_include_in_legend, idx2name, name2color, def paint_points_label(lidarseg_labels_filename: str, filter_lidarseg_labels: List[int], name2idx: Dict[str, int], colormap: Dict[str, List[int]]) -> np.ndarray: """ - + Paint each label in a pointcloud with the corresponding RGB value, and set the opacity of the labels to + be shown to 1 (the opacity of the rest will be set to 0); e.g.: + [30, 5, 12, 34, ...] ------> [[R30, G30, B30, 0], [R5, G5, B5, 1], [R34, G34, B34, 1], ...] + :param lidarseg_labels_filename: Path to the .bin file containing the labels. + :param filter_lidarseg_labels: The labels for which to set opacity to zero; this is to hide those points + thereby preventing them from being displayed. + :param name2idx: A dictionary containing the mapping from class names to class indices. + :param colormap: A dictionary containing the mapping from class names to RGB values. + :return: A numpy array which has length equal to the number of points in the pointcloud, and each value is + a RGBA array. """ # Load labels from .bin file.