diff --git a/python-sdk/nuscenes/nuscenes.py b/python-sdk/nuscenes/nuscenes.py index 347d481b..03523765 100644 --- a/python-sdk/nuscenes/nuscenes.py +++ b/python-sdk/nuscenes/nuscenes.py @@ -393,13 +393,16 @@ def box_velocity(self, sample_annotation_token: str, max_time_diff: float = 1.5) else: return pos_diff / time_diff - def get_sample_lidarseg_stats(self, sample_token: str, sort_counts: bool = True) -> None: + def get_sample_lidarseg_stats(self, sample_token: str, sort_counts: bool = True, + lidarseg_preds_bin_path: str = None) -> None: """ Print the number of points for each class in the lidar pointcloud of a sample. Classes with have no points in the pointcloud will not be printed. :param sample_token: Sample token. :param sort_counts: If True, the stats will be printed in ascending order of frequency; if False, the stats will be printed alphabetically according to class name. + :param lidarseg_preds_bin_path: A path to the .bin file which contains the user's lidar segmentation + predictions for the sample. """ assert hasattr(self, 'lidarseg'), 'Error: You have no lidarseg data; unable to get ' \ 'statistics for segmentation of the point cloud.' @@ -412,12 +415,28 @@ def get_sample_lidarseg_stats(self, sample_token: str, sort_counts: bool = True) assert ref_sd_record['is_key_frame'], 'Error: Only pointclouds which are keyframes have ' \ 'lidar segmentation labels. Rendering aborted.' - lidarseg_labels_filename = os.path.join(self.dataroot, 'lidarseg', ref_sd_token + '_lidarseg.bin') + if lidarseg_preds_bin_path: + lidarseg_labels_filename = lidarseg_preds_bin_path + assert os.path.exists(lidarseg_labels_filename), \ + 'Error: Unable to find {} to load the predictions for sample token {} ' \ + '(lidar sample data token {}) from.'.format(lidarseg_labels_filename, sample_token, ref_sd_token) + + header = '===== Statistics for ' + sample_token + ' (predictions) =====' + else: + assert len(self.lidarseg) > 0, 'Error: There are no ground truth labels found for nuScenes-lidarseg ' \ + 'for {}. Are you loading the test set? \nIf you want to see the sample ' \ + 'statistics for your predictions, pass a path to the appropriate .bin ' \ + 'file using the lidarseg_preds_bin_path argument.'.format(self.version) + lidar_sd_token = self.get('sample', sample_token)['data']['LIDAR_TOP'] + lidarseg_labels_filename = os.path.join(self.dataroot, + self.get('lidarseg', lidar_sd_token)['filename']) + + header = '===== Statistics for ' + sample_token + ' =====' + print(header) + points_label = np.fromfile(lidarseg_labels_filename, dtype=np.uint8) lidarseg_counts = get_stats(points_label, len(self.lidarseg_idx2name_mapping)) - print('===== Statistics for {} ====='.format(sample_token)) - lidarseg_counts_dict = dict() for i in range(len(lidarseg_counts)): lidarseg_counts_dict[self.lidarseg_idx2name_mapping[i]] = lidarseg_counts[i] @@ -432,7 +451,7 @@ def get_sample_lidarseg_stats(self, sample_token: str, sort_counts: bool = True) idx = get_key_from_value(self.lidarseg_idx2name_mapping, class_name) print('{:3} {:35} n={:12,}'.format(idx, class_name, count)) - print('======') + print('=' * len(header)) def list_categories(self) -> None: self.explorer.list_categories() @@ -456,7 +475,8 @@ def render_pointcloud_in_image(self, sample_token: str, dot_size: int = 5, point filter_lidarseg_labels: List = None, render_if_no_points: bool = True, show_lidarseg_legend: bool = False, - verbose: bool = True) -> None: + verbose: bool = True, + lidarseg_preds_bin_path: str = None) -> None: self.explorer.render_pointcloud_in_image(sample_token, dot_size, pointsensor_channel=pointsensor_channel, camera_channel=camera_channel, out_path=out_path, render_intensity=render_intensity, @@ -464,26 +484,31 @@ def render_pointcloud_in_image(self, sample_token: str, dot_size: int = 5, point filter_lidarseg_labels=filter_lidarseg_labels, render_if_no_points=render_if_no_points, show_lidarseg_legend=show_lidarseg_legend, - verbose=verbose) + verbose=verbose, + lidarseg_preds_bin_path=lidarseg_preds_bin_path) def render_sample(self, sample_token: str, box_vis_level: BoxVisibility = BoxVisibility.ANY, nsweeps: int = 1, out_path: str = None, show_lidarseg_labels: bool = False, - filter_lidarseg_labels: List = None, verbose: bool = True) -> None: + filter_lidarseg_labels: List = None, + lidarseg_preds_bin_path: str = None, verbose: bool = True) -> None: self.explorer.render_sample(sample_token, box_vis_level, nsweeps=nsweeps, out_path=out_path, show_lidarseg_labels=show_lidarseg_labels, - filter_lidarseg_labels=filter_lidarseg_labels, verbose=verbose) + filter_lidarseg_labels=filter_lidarseg_labels, + lidarseg_preds_bin_path=lidarseg_preds_bin_path, verbose=verbose) def render_sample_data(self, sample_data_token: str, with_anns: bool = True, box_vis_level: BoxVisibility = BoxVisibility.ANY, axes_limit: float = 40, ax: Axes = None, nsweeps: int = 1, out_path: str = None, underlay_map: bool = True, use_flat_vehicle_coordinates: bool = True, show_lidarseg_labels: bool = False, - filter_lidarseg_labels: List = None, verbose: bool = True) -> None: + filter_lidarseg_labels: List = None, + lidarseg_preds_bin_path: str = None, verbose: bool = True) -> None: self.explorer.render_sample_data(sample_data_token, with_anns, box_vis_level, axes_limit, ax, nsweeps=nsweeps, out_path=out_path, underlay_map=underlay_map, use_flat_vehicle_coordinates=use_flat_vehicle_coordinates, show_lidarseg_labels=show_lidarseg_labels, - filter_lidarseg_labels=filter_lidarseg_labels, verbose=verbose) + filter_lidarseg_labels=filter_lidarseg_labels, + lidarseg_preds_bin_path=lidarseg_preds_bin_path, verbose=verbose) def render_annotation(self, sample_annotation_token: str, margin: float = 10, view: np.ndarray = np.eye(4), box_vis_level: BoxVisibility = BoxVisibility.ANY, out_path: str = None, @@ -509,20 +534,24 @@ def render_egoposes_on_map(self, log_location: str, scene_tokens: List = None, o def render_camera_channel_with_pointclouds(self, scene_token: str, camera_channel: str, out_folder: str = None, filter_lidarseg_labels: Iterable[int] = None, render_if_no_points: bool = True, verbose=True, - imsize: Tuple[int, int] = (640, 360), freq: float = 2) -> None: - self.explorer.render_camera_channel_with_pointclouds(scene_token, camera_channel, out_folder, - filter_lidarseg_labels, - render_if_no_points, verbose, - imsize, freq) + imsize: Tuple[int, int] = (640, 360), freq: float = 2, + lidarseg_preds_folder: str = None) -> None: + self.explorer.render_camera_channel_with_pointclouds(scene_token, camera_channel, out_folder=out_folder, + filter_lidarseg_labels=filter_lidarseg_labels, + render_if_no_points=render_if_no_points, verbose=verbose, + imsize=imsize, freq=freq, + lidarseg_preds_folder=lidarseg_preds_folder) def render_scene_with_pointclouds_for_all_cameras(self, scene_token: str, out_path: str = None, filter_lidarseg_labels: Iterable[int] = None, imsize: Tuple[int, int] = (640, 360), freq: float = 2, - verbose: bool = True) -> None: - self.explorer.render_scene_with_pointclouds_for_all_cameras(scene_token, out_path, - filter_lidarseg_labels, - imsize, freq, - verbose) + verbose: bool = True, + lidarseg_preds_folder: str = None) -> None: + self.explorer.render_scene_with_pointclouds_for_all_cameras(scene_token, out_path=out_path, + filter_lidarseg_labels=filter_lidarseg_labels, + imsize=imsize, freq=freq, + verbose=verbose, + lidarseg_preds_folder=lidarseg_preds_folder) class NuScenesExplorer: @@ -582,8 +611,8 @@ def list_lidarseg_categories(self) -> None: lidarseg_counts = [0] * len(self.nusc.lidarseg_idx2name_mapping) for record_lidarseg in self.nusc.lidarseg: - lidarseg_labels_filename = osp.join(self.nusc.dataroot, 'lidarseg', - record_lidarseg['sample_data_token'] + '_lidarseg.bin') + lidarseg_labels_filename = osp.join(self.nusc.dataroot, record_lidarseg['filename']) + points_label = np.fromfile(lidarseg_labels_filename, dtype=np.uint8) indices = np.bincount(points_label) ii = np.nonzero(indices)[0] @@ -665,7 +694,7 @@ def map_pointcloud_to_image(self, render_intensity: bool = False, show_lidarseg_labels: bool = False, filter_lidarseg_labels: List = None, - render_if_no_points: bool = True) -> Tuple: + lidarseg_preds_bin_path: str = None) -> Tuple: """ Given a point sensor (lidar/radar) token and camera sample_data token, load point-cloud and map it to the image plane. @@ -676,7 +705,8 @@ def map_pointcloud_to_image(self, :param show_lidarseg_labels: Whether to render lidar intensity instead of point depth. :param filter_lidarseg_labels: Only show lidar points which belong to the given list of classes. If None or the list is empty, all classes will be displayed. - :param render_if_no_points: Whether to render if there are no points (e.g. after filtering) in the image. + :param lidarseg_preds_bin_path: A path to the .bin file which contains the user's lidar segmentation + predictions for the sample. :return (pointcloud , coloring , image ). """ @@ -737,15 +767,33 @@ def map_pointcloud_to_image(self, elif show_lidarseg_labels: assert pointsensor['sensor_modality'] == 'lidar', 'Error: Can only render lidarseg labels for lidar, ' \ 'not %s!' % pointsensor['sensor_modality'] - lidarseg_labels_filename = osp.join(self.nusc.dataroot, 'lidarseg', pointsensor_token + '_lidarseg.bin') - points_label = np.fromfile(lidarseg_labels_filename, dtype=np.uint8) - # Create colormap for the lidarseg labels. - num_classes = len(self.nusc.lidarseg_idx2name_mapping) - colormap = get_arbitrary_colormap(num_classes) - if filter_lidarseg_labels: - colormap = filter_colormap(colormap, filter_lidarseg_labels) - coloring = colormap[points_label] + if lidarseg_preds_bin_path: + sample_token = self.nusc.get('sample_data', pointsensor_token)['sample_token'] + lidarseg_labels_filename = lidarseg_preds_bin_path + assert os.path.exists(lidarseg_labels_filename), \ + 'Error: Unable to find {} to load the predictions for sample token {} (lidar ' \ + 'sample data token {}) from.'.format(lidarseg_labels_filename, sample_token, pointsensor_token) + else: + if len(self.nusc.lidarseg) > 0: # Ensure lidarseg.json is not empty (e.g. in case of v1.0-test). + lidarseg_labels_filename = osp.join(self.nusc.dataroot, + self.nusc.get('lidarseg', pointsensor_token)['filename']) + else: + lidarseg_labels_filename = None + + if lidarseg_labels_filename: + points_label = np.fromfile(lidarseg_labels_filename, dtype=np.uint8) + + # Create colormap for the lidarseg labels. + num_classes = len(self.nusc.lidarseg_idx2name_mapping) + colormap = get_arbitrary_colormap(num_classes) + if filter_lidarseg_labels: + colormap = filter_colormap(colormap, filter_lidarseg_labels) + coloring = colormap[points_label] + else: + coloring = depths + print('Warning: There are no lidarseg labels in {}. Points will be colored according to distance ' + 'from the ego vehicle instead.'.format(self.nusc.version)) else: # Retrieve the color from the depth. coloring = depths @@ -765,12 +813,6 @@ def map_pointcloud_to_image(self, points = points[:, mask] coloring = coloring[mask] - # Prevent rendering images which have no lidarseg labels in it. To check if there are no lidarseg - # labels in an image, we check if any column in the coloring is all zeros (the alpha column will - # be all zeroes if so). - if show_lidarseg_labels and not render_if_no_points and (~coloring.any(axis=0)).any(): - return None, None, None - return points, coloring, im def render_pointcloud_in_image(self, @@ -785,7 +827,8 @@ def render_pointcloud_in_image(self, ax: Axes = None, render_if_no_points: bool = True, show_lidarseg_legend: bool = False, - verbose: bool = True): + verbose: bool = True, + lidarseg_preds_bin_path: str = None): """ Scatter-plots a point-cloud on top of image. :param sample_token: Sample token. @@ -800,6 +843,9 @@ def render_pointcloud_in_image(self, :param render_if_no_points: Whether to render if there are no points (e.g. after filtering) in the image. :param show_lidarseg_legend: Whether to display the legend for the lidarseg labels in the frame. :param verbose: Whether to display the image in a window. + :param lidarseg_preds_bin_path: A path to the .bin file which contains the user's lidar segmentation + predictions for the sample. + """ sample_record = self.nusc.get('sample', sample_token) @@ -811,7 +857,14 @@ def render_pointcloud_in_image(self, render_intensity=render_intensity, show_lidarseg_labels=show_lidarseg_labels, filter_lidarseg_labels=filter_lidarseg_labels, - render_if_no_points=render_if_no_points) + lidarseg_preds_bin_path=lidarseg_preds_bin_path) + + # Prevent rendering images which have no lidarseg labels in them (e.g. the classes in the filter chosen by + # the users do not appear within the image). To check if there are no lidarseg labels belonging to the desired + # classes in an image, we check if any column in the coloring is all zeros (the alpha column will be all + # zeroes if so). + if show_lidarseg_labels and not render_if_no_points and (~coloring.any(axis=0)).any(): + points, coloring, im = None, None, None if pointsensor_channel == 'LIDAR_TOP': # Prevent rendering images which have no lidarseg labels. @@ -823,7 +876,11 @@ def render_pointcloud_in_image(self, # Init axes. if ax is None: - _, ax = plt.subplots(1, 1, figsize=(9, 16)) + fig, ax = plt.subplots(1, 1, figsize=(9, 16)) + if lidarseg_preds_bin_path: + fig.canvas.set_window_title(sample_token + '(predictions)') + else: + fig.canvas.set_window_title(sample_token) else: # Set title on if rendering as part of render_sample. ax.set_title(camera_channel) ax.imshow(im) @@ -857,6 +914,7 @@ def render_sample(self, out_path: str = None, show_lidarseg_labels: bool = False, filter_lidarseg_labels: List = None, + lidarseg_preds_bin_path: str = None, verbose: bool = True) -> None: """ Render all LIDAR and camera sample_data in sample along with annotations. @@ -866,6 +924,8 @@ def render_sample(self, :param out_path: Optional path to save the rendered figure to disk. :param show_lidarseg_labels: Whether to show lidar segmentations labels or not. :param filter_lidarseg_labels: Only show lidar points which belong to the given list of classes. + :param lidarseg_preds_bin_path: A path to the .bin file which contains the user's lidar segmentation + predictions for the sample. :param verbose: Whether to show the rendered sample in a window or not. """ record = self.nusc.get('sample', token) @@ -905,7 +965,9 @@ def render_sample(self, for (_, sd_token), ax in zip(lidar_data.items(), axes.flatten()[num_radar_plots:]): self.render_sample_data(sd_token, box_vis_level=box_vis_level, ax=ax, nsweeps=nsweeps, show_lidarseg_labels=show_lidarseg_labels, - filter_lidarseg_labels=filter_lidarseg_labels, verbose=False) + filter_lidarseg_labels=filter_lidarseg_labels, + lidarseg_preds_bin_path=lidarseg_preds_bin_path, + verbose=False) # Plot cameras in separate subplots. for (_, sd_token), ax in zip(camera_data.items(), axes.flatten()[num_radar_plots + num_lidar_plots:]): @@ -924,7 +986,8 @@ def render_sample(self, camera_channel=sensor_channel, show_lidarseg_labels=show_lidarseg_labels, filter_lidarseg_labels=filter_lidarseg_labels, - ax=ax, verbose=False) + ax=ax, verbose=False, + lidarseg_preds_bin_path=lidarseg_preds_bin_path) # Change plot settings and write to disk. axes.flatten()[-1].axis('off') @@ -1007,6 +1070,7 @@ def render_sample_data(self, use_flat_vehicle_coordinates: bool = True, show_lidarseg_labels: bool = False, filter_lidarseg_labels: List = None, + lidarseg_preds_bin_path: str = None, verbose: bool = True) -> None: """ Render sample data onto axis. @@ -1026,6 +1090,8 @@ def render_sample_data(self, to False, the colors of the lidar data represent the distance from the center of the ego vehicle. :param filter_lidarseg_labels: Only show lidar points which belong to the given list of classes. If None or the list is empty, all classes will be displayed. + :param lidarseg_preds_bin_path: A path to the .bin file which contains the user's lidar segmentation + predictions for the sample. :param verbose: Whether to display the image after it is rendered. """ # Get sensor modality. @@ -1110,16 +1176,32 @@ def render_sample_data(self, dists = np.sqrt(np.sum(pc.points[:2, :] ** 2, axis=0)) if sensor_modality == 'lidar' and show_lidarseg_labels: # Load labels for pointcloud. - lidarseg_labels_filename = osp.join(self.nusc.dataroot, 'lidarseg', - sample_data_token + '_lidarseg.bin') - points_label = np.fromfile(lidarseg_labels_filename, dtype=np.uint8) - - # Create colormap for the lidarseg labels. - num_classes = len(self.nusc.lidarseg_idx2name_mapping) - colormap = get_arbitrary_colormap(num_classes) - if filter_lidarseg_labels: - colormap = filter_colormap(colormap, filter_lidarseg_labels) - colors = colormap[points_label] + if lidarseg_preds_bin_path: + sample_token = self.nusc.get('sample_data', sample_data_token)['sample_token'] + lidarseg_labels_filename = lidarseg_preds_bin_path + assert os.path.exists(lidarseg_labels_filename), \ + 'Error: Unable to find {} to load the predictions for sample token {} (lidar ' \ + 'sample data token {}) from.'.format(lidarseg_labels_filename, sample_token, sample_data_token) + else: + if len(self.nusc.lidarseg) > 0: # Ensure lidarseg.json is not empty (e.g. in case of v1.0-test). + lidarseg_labels_filename = osp.join(self.nusc.dataroot, + self.nusc.get('lidarseg', sample_data_token)['filename']) + else: + lidarseg_labels_filename = None + + if lidarseg_labels_filename: + points_label = np.fromfile(lidarseg_labels_filename, dtype=np.uint8) + + # Create colormap for the lidarseg labels. + num_classes = len(self.nusc.lidarseg_idx2name_mapping) + colormap = get_arbitrary_colormap(num_classes) + if filter_lidarseg_labels: + colormap = filter_colormap(colormap, filter_lidarseg_labels) + colors = colormap[points_label] + else: + colors = np.minimum(1, dists / axes_limit / np.sqrt(2)) + print('Warning: There are no lidarseg labels in {}. Points will be colored according to distance ' + 'from the ego vehicle instead.'.format(self.nusc.version)) else: colors = np.minimum(1, dists / axes_limit / np.sqrt(2)) point_scale = 0.2 if sensor_modality == 'lidar' else 3.0 @@ -1181,7 +1263,8 @@ def render_sample_data(self, raise ValueError("Error: Unknown sensor modality!") ax.axis('off') - ax.set_title(sd_record['channel']) + ax.set_title('{} {labels_type}'.format( + sd_record['channel'], labels_type='(predictions)' if lidarseg_preds_bin_path else '')) ax.set_aspect('equal') if out_path is not None: @@ -1596,7 +1679,8 @@ def render_camera_channel_with_pointclouds(self, scene_token: str, camera_channe render_if_no_points: bool = True, verbose: bool = True, imsize: Tuple[int, int] = (640, 360), - freq: float = 2) -> None: + freq: float = 2, + lidarseg_preds_folder: str = None) -> None: """ Renders a full scene with labelled lidar pointclouds for a particular camera channel. :param scene_token: Unique identifier of scene to render. @@ -1611,6 +1695,9 @@ def render_camera_channel_with_pointclouds(self, scene_token: str, camera_channe :param freq: Display frequency (Hz). :param render_if_no_points: Whether to render if there are no points (e.g. after filtering) in the image. :param verbose: Whether to show the frames as they are being rendered. + :param lidarseg_preds_folder: A path to the folder which contains the user's lidar segmentation predictions for + the scene. The naming convention of each .bin file in the folder should be + named in this format: _lidarseg.bin. """ valid_channels = ['CAM_FRONT_LEFT', 'CAM_FRONT', 'CAM_FRONT_RIGHT', @@ -1639,7 +1726,8 @@ def render_camera_channel_with_pointclouds(self, scene_token: str, camera_channe # Open CV init. if verbose: - name = '{}: {} (Space to pause, ESC to exit)'.format(scene_record['name'], camera_channel) + name = '{}: {} {labels_type} (Space to pause, ESC to exit)'.format( + scene_record['name'], camera_channel, labels_type="(predictions)" if lidarseg_preds_folder else "") cv2.namedWindow(name) cv2.moveWindow(name, 0, 0) else: @@ -1665,11 +1753,23 @@ def render_camera_channel_with_pointclouds(self, scene_token: str, camera_channe pointsensor_token = sample_record['data']['LIDAR_TOP'] camera_token = sample_record['data'][camera_channel] + if lidarseg_preds_folder: + lidarseg_preds_bin_path = osp.join(lidarseg_preds_folder, pointsensor_token + '_lidarseg.bin') + else: + lidarseg_preds_bin_path = None + points, coloring, im = self.map_pointcloud_to_image(pointsensor_token, camera_token, render_intensity=False, show_lidarseg_labels=True, filter_lidarseg_labels=filter_lidarseg_labels, - render_if_no_points=render_if_no_points) + lidarseg_preds_bin_path=lidarseg_preds_bin_path) + + # Prevent rendering images which have no lidarseg labels in them (e.g. the classes in the filter chosen by + # the users do not appear within the image). To check if there are no lidarseg labels belonging to the + # desired classes in an image, we check if any column in the coloring is all zeros (the alpha column will + # be all zeroes if so). + if not render_if_no_points and (~coloring.any(axis=0)).any(): + points, coloring, im = None, None, None if im is not None: mat = plt_to_cv2(points, coloring, im, imsize) @@ -1709,7 +1809,8 @@ def render_camera_channel_with_pointclouds(self, scene_token: str, camera_channe def render_scene_with_pointclouds_for_all_cameras(self, scene_token: str, out_path: str = None, filter_lidarseg_labels: Iterable[int] = None, imsize: Tuple[int, int] = (640, 360), freq: float = 2, - verbose: bool = True) -> None: + verbose: bool = True, + lidarseg_preds_folder: str = None) -> None: """ Renders a full scene with all camera channels and the lidar segmentation labels for each camera. :param scene_token: Unique identifier of scene to render. @@ -1719,6 +1820,9 @@ def render_scene_with_pointclouds_for_all_cameras(self, scene_token: str, out_pa :param freq: Display frequency (Hz). :param imsize: Size of image to render. The larger the slower this will run. :param verbose: Whether to show the frames as they are being rendered. + :param lidarseg_preds_folder: A path to the folder which contains the user's lidar segmentation predictions for + the scene. The naming convention of each .bin file in the folder should be + named in this format: _lidarseg.bin. """ assert imsize[0] / imsize[1] == 16 / 9, "Aspect ratio should be 16/9." @@ -1741,7 +1845,8 @@ def render_scene_with_pointclouds_for_all_cameras(self, scene_token: str, out_pa 'CAM_BACK_RIGHT': (2 * imsize[0], imsize[1]), } - window_name = '{} (Space to pause, ESC to exit)'.format(scene_record['name']) + window_name = '{} {labels_type} (Space to pause, ESC to exit)'.format( + scene_record['name'], labels_type="(predictions)" if lidarseg_preds_folder else "") cv2.namedWindow(window_name) cv2.moveWindow(window_name, 0, 0) @@ -1767,13 +1872,18 @@ def render_scene_with_pointclouds_for_all_cameras(self, scene_token: str, out_pa pointsensor_token = sample_record['data']['LIDAR_TOP'] camera_token = sample_record['data'][camera_channel] + if lidarseg_preds_folder: + lidarseg_preds_bin_path = osp.join(lidarseg_preds_folder, pointsensor_token + '_lidarseg.bin') + else: + lidarseg_preds_bin_path = None + # render_if_no_points has to be true or the rendering will fail as some cameras will # have points and others will not. points, coloring, im = self.map_pointcloud_to_image(pointsensor_token, camera_token, render_intensity=False, show_lidarseg_labels=True, filter_lidarseg_labels=filter_lidarseg_labels, - render_if_no_points=True) + lidarseg_preds_bin_path=lidarseg_preds_bin_path) if im is not None: mat = plt_to_cv2(points, coloring, im, imsize) diff --git a/python-sdk/tutorials/lidarseg_render_scene.gif b/python-sdk/tutorials/lidarseg_render_scene.gif deleted file mode 100644 index ee6c6d7e..00000000 Binary files a/python-sdk/tutorials/lidarseg_render_scene.gif and /dev/null differ diff --git a/python-sdk/tutorials/lidarseg_render_scene_all_cams.gif b/python-sdk/tutorials/lidarseg_render_scene_all_cams.gif deleted file mode 100644 index feb0b0b7..00000000 Binary files a/python-sdk/tutorials/lidarseg_render_scene_all_cams.gif and /dev/null differ diff --git a/python-sdk/tutorials/nuscenes_lidarseg_tutorial.ipynb b/python-sdk/tutorials/nuscenes_lidarseg_tutorial.ipynb index e4d3a439..01e4a742 100644 --- a/python-sdk/tutorials/nuscenes_lidarseg_tutorial.ipynb +++ b/python-sdk/tutorials/nuscenes_lidarseg_tutorial.ipynb @@ -16,21 +16,23 @@ "metadata": {}, "source": [ "## Setup\n", - "To install the nuScenes-lidarseg expansion, please download the dataset from https://www.nuscenes.org/download. Upon unzipping, you should see three items:\n", - "- ```lidarseg``` \n", - "- ```lidarseg_category.json```\n", - "- ```lidarseg.json```\n", + "To install the nuScenes-lidarseg expansion, please download the dataset from https://www.nuscenes.org/download. Upon unzipping, you should see two items:\n", + "- `lidarseg_v1.0-mini` \n", + "- `v1.0-mini`\n", "\n", - "Copy the ```lidarseg``` folder (this contains .bin files; each .bin file is an annotation for a pointcloud) into ```/data/sets/nuscenes/```, and copy ```lidarseg_category.json``` and ```lidarseg.json``` into ```/data/sets/nuscenes/v1.0-mini```.\n", + "Copy both the `lidarseg_v1.0-mini` folder (this contains `.bin` files; each .bin file contains the label of each point in the point cloud) and `v1.0-mini` into `/data/sets/nuscenes`. If you already have an existing `v1.0-mini` folder from nuScenes, it is safe to replace it as the `v1.0-mini` from nuScenes-lidarseg will have the required files as well. \n", "\n", - "Essentially, your ```/data/sets/nuscenes``` should end up looking like this:\n", + "If you have downloaded `v1.0-trainval` and `v1.0-test` for nuScenes-lidarseg as well, place (or replace) them in `/data/sets/nuscenes`. \n", + "\n", + "Essentially, your `/data/sets/nuscenes` folder should end up looking like this:\n", "```\n", "└── nuscenes \n", " ├── Usual nuscenes folders (i.e. samples, sweep)\n", " │\n", - " ├── lidarseg <- Contains the .bin files \n", - " │ (each .bin file contains the label of each point in the point cloud)\n", - " └── v1.0-mini\n", + " ├── lidarseg_v1.0-{mini, trainval} <- Contains the .bin files (Note that v1.0-test does not \n", + " │ have any .bin files associated with it.\n", + " │\n", + " └── v1.0-{mini, trainval, test}\n", " ├── Usual files (e.g. attribute.json, calibrated_sensor.json etc.) \n", " ├── lidarseg.json <- contains the mapping of each .bin file to the token \n", " └── lidarseg_category.json <- contains the categories of the labels\n", @@ -62,7 +64,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "As you can see, you do not need any extra libraries to use nuScenes-lidarseg. The original nuScenes devkit which you are familiar with has been extended so tha you can use it seamless with nuScenes-lidarseg." + "As you can see, you do not need any extra libraries to use nuScenes-lidarseg. The original nuScenes devkit which you are familiar with has been extended so that you can use it seamlessly with nuScenes-lidarseg." ] }, { @@ -274,8 +276,6 @@ "source": [ "We then pass the scene token into ```render_camera_channel_with_pointclouds``` indicating that we are only interested in construction vehicles and man-made objects.\n", "\n", - "\n", - "\n", "(Note: the following code is commented out as it crashes in Jupyter notebooks.)" ] }, @@ -334,7 +334,7 @@ "source": [ "# nusc.render_camera_channel_with_pointclouds(my_scene['token'], 'CAM_BACK',\n", "# filter_lidarseg_labels=[6],\n", - "# render_if_no_points=False,\n", + "# render_if_no_points=True,\n", "# verbose=True,\n", "# imsize=(1280, 720),\n", "# out_folder=os.path.expanduser('~/Desktop/my_rendered_scene.avi'))" @@ -347,8 +347,6 @@ "## Render a scene for all cameras with lidarseg labels\n", "You can also render the entire scene for all cameras at once with the lidarseg labels as a video. Let's say in this case, we are interested in points belonging to driveable surfaces and cars.\n", "\n", - "\n", - "\n", "(Note: the following code is commented out as it crashes in Jupyter notebooks.)" ] }, @@ -363,6 +361,75 @@ "# out_path=os.path.expanduser('~/Desktop/my_rendered_scene.avi'))" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Visualizing LIDAR segmentation predictions\n", + "In all the above functions, the labels of the LIDAR pointcloud which have been rendered are the ground truth. If you have trained a model to segment LIDAR pointclouds and have run it on the nuScenes-lidarseg dataset, you can visualize your model's predictions with nuScenes-lidarseg as well!\n", + "\n", + "You simply need to pass the path to the .bin file where your predictions for the given sample are to `lidarseg_preds_bin_path` for these functions:\n", + "- `render_sample_data`\n", + "- `render_pointcloud_in_image`\n", + "- `render_sample` \n", + "\n", + "For example, let's assume the predictions for `my_sample` is stored at `/data/sets/nuscenes/lidarseg` with the format `_lidarseg.bin`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "my_sample = nusc.sample[87]\n", + "sample_data_token = my_sample['data']['LIDAR_TOP']\n", + "my_predictions_bin_file = os.path.join('/data/sets/nuscenes/lidarseg_v1.0-mini', sample_data_token + '_lidarseg.bin')\n", + "\n", + "nusc.render_pointcloud_in_image(my_sample['token'],\n", + " pointsensor_channel='LIDAR_TOP',\n", + " camera_channel='CAM_BACK',\n", + " render_intensity=False,\n", + " show_lidarseg_labels=True,\n", + " filter_lidarseg_labels=[3, 4, 5],\n", + " show_lidarseg_legend=True,\n", + " verbose=True,\n", + " lidarseg_preds_bin_path=my_predictions_bin_file)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For these functions that render an entire scene, you will need to pass the path to the folder which contains the .bin files for each sample in a scene to `lidarseg_preds_folder`:\n", + "- `render_camera_channel_with_pointclouds`\n", + "- `render_scene_with_pointclouds_for_all_cameras`\n", + "\n", + "Pay special attention that **each set of predictions in the folder _must_ be a `.bin` file and named as `_lidarseg.bin`**.\n", + "\n", + "(Note: the following code is commented out as it crashes in Jupyter notebooks.)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# my_scene = nusc.scene[0]\n", + "# my_folder_of_predictions = '/data/sets/nuscenes/lidarseg_v1.0-mini'\n", + "\n", + "# nusc.render_camera_channel_with_pointclouds(my_scene['token'], \n", + "# 'CAM_BACK', \n", + "# filter_lidarseg_labels=[6, 36],\n", + "# render_if_no_points=True, \n", + "# verbose=True, \n", + "# imsize=(1280, 720),\n", + "# lidarseg_preds_folder=my_folder_of_predictions)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -388,7 +455,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.6" + "version": "3.7.7" } }, "nbformat": 4,