Skip to content

Commit

Permalink
Improve aesthetics for render_scene_with_pointclouds_for_all_cameras (#…
Browse files Browse the repository at this point in the history 
…412)

* Flip back cams horizontally for aesthetics

* Explicitly set margins to zero and turn axes off

* Prevent final frames from showing up in notebook if user stops render

* Allow render_scene_with_pointclouds_for_all_cameras to output frames as images

* Update indices in tutorial examples

* Style changes

* Clear memory in render_camera_channel_with_pointclouds if user stops rendering

* Graceful exit if users stops rendering

* Change random seed for color scheme

* Style edits
  • Loading branch information
@whyekit-motional
whyekit-motional authored Jun 12, 2020
1 parent 3c6e178 commit 606f387
Show file tree
Hide file tree
Showing 3 changed files with 60 additions and 20 deletions.
5 changes: 4 additions & 1 deletion python-sdk/nuscenes/lidarseg/lidarseg_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@ def plt_to_cv2(points: np.array, coloring: np.array, im, imsize: Tuple[int, int]
ax = plt.Axes(fig, [0., 0., 1., 1.])
fig.add_axes(ax)

ax.axis('off')
ax.margins(0, 0)

ax.imshow(im)
ax.scatter(points[0, :], points[1, :], c=coloring, s=5)

Expand Down Expand Up @@ -121,7 +124,7 @@ def get_colormap() -> np.ndarray:
return colormap


def get_arbitrary_colormap(num_classes: int, random_seed: int = 2020) -> np.ndarray:
def get_arbitrary_colormap(num_classes: int, random_seed: int = 93) -> np.ndarray:
"""
Create an arbitrary RGB colormap. Note that the RGB values are normalized between 0 and 1, not 0 and 255.
:param num_classes: Number of colors to create.
Expand Down
53 changes: 45 additions & 8 deletions python-sdk/nuscenes/nuscenes.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1795,6 +1795,12 @@ def render_camera_channel_with_pointclouds(self, scene_token: str, camera_channe
key = cv2.waitKey()

if key == 27: # if ESC is pressed, exit.
plt.close('all') # To prevent figures from accumulating in memory.
# If rendering is stopped halfway, save whatever has been rendered so far into a video
# (if save_as_vid = True).
if save_as_vid:
out.write(mat)
out.release()
cv2.destroyAllWindows()
break

Expand Down Expand Up @@ -1827,7 +1833,10 @@ def render_scene_with_pointclouds_for_all_cameras(self, scene_token: str, out_pa
"""
Renders a full scene with all camera channels and the lidar segmentation labels for each camera.
:param scene_token: Unique identifier of scene to render.
:param out_path: Optional path to write a video file of the rendered frames.
:param out_path: Optional path to save the rendered figure to disk. The filename of each image will be
same as the original image's. If .avi is specified (e.g. '~/Desktop/my_rendered_scene.avi),
a video will be written instead of saving individual frames as images. Each image name wil
follow this format: <0-scene_number>_<frame_number>.jpg
:param filter_lidarseg_labels: Only show lidar points which belong to the given list of classes. If None
or the list is empty, all classes will be displayed.
:param freq: Display frequency (Hz).
Expand All @@ -1839,6 +1848,15 @@ def render_scene_with_pointclouds_for_all_cameras(self, scene_token: str, out_pa
"""
assert imsize[0] / imsize[1] == 16 / 9, "Aspect ratio should be 16/9."

if out_path is not None:
if os.path.splitext(out_path)[-1] == '.avi':
save_as_vid = True
else:
assert os.path.isdir(out_path), 'Error: {} does not exist.'.format(out_path)
save_as_vid = False
else:
save_as_vid = False

# Get records from DB.
scene_record = self.nusc.get('scene', scene_token)

Expand All @@ -1858,18 +1876,22 @@ def render_scene_with_pointclouds_for_all_cameras(self, scene_token: str, out_pa
'CAM_BACK_RIGHT': (2 * imsize[0], imsize[1]),
}

window_name = '{} {labels_type} (Space to pause, ESC to exit)'.format(
scene_record['name'], labels_type="(predictions)" if lidarseg_preds_folder else "")
cv2.namedWindow(window_name)
cv2.moveWindow(window_name, 0, 0)
horizontal_flip = ['CAM_BACK_LEFT', 'CAM_BACK', 'CAM_BACK_RIGHT'] # Flip these for aesthetic reasons.

if verbose:
window_name = '{} {labels_type} (Space to pause, ESC to exit)'.format(
scene_record['name'], labels_type="(predictions)" if lidarseg_preds_folder else "")
cv2.namedWindow(window_name)
cv2.moveWindow(window_name, 0, 0)
else:
window_name = None

slate = np.ones((2 * imsize[1], 3 * imsize[0], 3), np.uint8)
save_as_vid = False
if out_path is not None:

if save_as_vid:
assert os.path.splitext(out_path)[-1] == '.avi', 'Error: Video can only be saved in .avi format.'
fourcc = cv2.VideoWriter_fourcc(*'MJPG')
out = cv2.VideoWriter(out_path, fourcc, freq, slate.shape[1::-1])
save_as_vid = True
else:
out = None

Expand All @@ -1880,6 +1902,7 @@ def render_scene_with_pointclouds_for_all_cameras(self, scene_token: str, out_pa
keep_looping = False

sample_record = self.nusc.get('sample', current_token)
filename = '0' + scene_record['name'][5:] + '_{:02d}.jpg'.format(i)

for camera_channel in layout:
pointsensor_token = sample_record['data']['LIDAR_TOP']
Expand All @@ -1901,6 +1924,10 @@ def render_scene_with_pointclouds_for_all_cameras(self, scene_token: str, out_pa
if im is not None:
mat = plt_to_cv2(points, coloring, im, imsize)

if camera_channel in horizontal_flip:
# Flip image horizontally.
mat = cv2.flip(mat, 1)

slate[layout[camera_channel][1]: layout[camera_channel][1] + imsize[1],
layout[camera_channel][0]:layout[camera_channel][0] + imsize[0], :] = mat

Expand All @@ -1912,13 +1939,23 @@ def render_scene_with_pointclouds_for_all_cameras(self, scene_token: str, out_pa
key = cv2.waitKey()

if key == 27: # if ESC is pressed, exit.
plt.close('all') # To prevent figures from accumulating in memory.
# If rendering is stopped halfway, save whatever has been rendered so far into a video
# (if save_as_vid = True).
if save_as_vid:
out.write(slate)
out.release()
cv2.destroyAllWindows()
break

plt.close('all') # To prevent figures from accumulating in memory.

if save_as_vid:
out.write(slate)
elif out_path:
cv2.imwrite(os.path.join(out_path, filename), slate)
else:
pass

next_token = sample_record['next']
current_token = next_token
Expand Down
22 changes: 11 additions & 11 deletions python-sdk/tutorials/nuscenes_lidarseg_tutorial.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@
"nusc.render_sample_data(sample_data_token,\n",
" with_anns=False,\n",
" show_lidarseg_labels=True,\n",
" filter_lidarseg_labels=[3, 4, 5])"
" filter_lidarseg_labels=[9, 14, 18])"
]
},
{
Expand Down Expand Up @@ -184,7 +184,7 @@
" camera_channel='CAM_BACK',\n",
" render_intensity=False,\n",
" show_lidarseg_labels=True,\n",
" filter_lidarseg_labels=[3, 4, 5],\n",
" filter_lidarseg_labels=[9, 14, 18],\n",
" show_lidarseg_legend=True,\n",
" verbose=True)"
]
Expand All @@ -207,7 +207,7 @@
" camera_channel='CAM_BACK',\n",
" render_intensity=False,\n",
" show_lidarseg_labels=True,\n",
" filter_lidarseg_labels=[30, 15],\n",
" filter_lidarseg_labels=[21, 11],\n",
" show_lidarseg_legend=True,\n",
" verbose=True,\n",
" render_if_no_points=False)"
Expand Down Expand Up @@ -236,7 +236,7 @@
"source": [
"nusc.render_sample(my_sample['token'],\n",
" show_lidarseg_labels=True,\n",
" filter_lidarseg_labels=[3, 4, 5],\n",
" filter_lidarseg_labels=[9, 14, 18],\n",
" verbose=True)"
]
},
Expand Down Expand Up @@ -282,7 +282,7 @@
"# import os\n",
"# nusc.render_camera_channel_with_pointclouds(my_scene['token'], \n",
"# 'CAM_BACK', \n",
"# filter_lidarseg_labels=[6, 36],\n",
"# filter_lidarseg_labels=[15, 30],\n",
"# render_if_no_points=True, \n",
"# verbose=True, \n",
"# imsize=(1280, 720))"
Expand All @@ -304,7 +304,7 @@
"outputs": [],
"source": [
"# nusc.render_camera_channel_with_pointclouds(my_scene['token'], 'CAM_BACK',\n",
"# filter_lidarseg_labels=[6, 36],\n",
"# filter_lidarseg_labels=[15, 30],\n",
"# render_if_no_points=True,\n",
"# verbose=True,\n",
"# imsize=(1280, 720),\n",
Expand All @@ -327,7 +327,7 @@
"outputs": [],
"source": [
"# nusc.render_camera_channel_with_pointclouds(my_scene['token'], 'CAM_BACK',\n",
"# filter_lidarseg_labels=[6],\n",
"# filter_lidarseg_labels=[15],\n",
"# render_if_no_points=True,\n",
"# verbose=True,\n",
"# imsize=(1280, 720),\n",
Expand All @@ -351,7 +351,7 @@
"outputs": [],
"source": [
"# nusc.render_scene_with_pointclouds_for_all_cameras(my_scene['token'], \n",
"# filter_lidarseg_labels=[32, 1],\n",
"# filter_lidarseg_labels=[26, 9],\n",
"# out_path=os.path.expanduser('~/Desktop/my_rendered_scene.avi'))"
]
},
Expand Down Expand Up @@ -388,7 +388,7 @@
" camera_channel='CAM_BACK',\n",
" render_intensity=False,\n",
" show_lidarseg_labels=True,\n",
" filter_lidarseg_labels=[3, 4, 5],\n",
" filter_lidarseg_labels=[9, 14, 18],\n",
" show_lidarseg_legend=True,\n",
" verbose=True,\n",
" lidarseg_preds_bin_path=my_predictions_bin_file)"
Expand Down Expand Up @@ -418,7 +418,7 @@
"\n",
"# nusc.render_camera_channel_with_pointclouds(my_scene['token'], \n",
"# 'CAM_BACK', \n",
"# filter_lidarseg_labels=[6, 36],\n",
"# filter_lidarseg_labels=[15, 30],\n",
"# render_if_no_points=True, \n",
"# verbose=True, \n",
"# imsize=(1280, 720),\n",
Expand Down Expand Up @@ -450,7 +450,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.7"
"version": "3.7.6"
}
},
"nbformat": 4,
Expand Down

0 comments on commit 606f387

Please sign in to comment.