Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve aesthetics for render_scene_with_pointclouds_for_all_cameras #412

Merged
merged 10 commits into from
Jun 12, 2020
Merged

Improve aesthetics for render_scene_with_pointclouds_for_all_cameras #412

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
71843ae
Flip back cams horizontally for aesthetics
Jun 10, 2020
8b5c514
Explicitly set margins to zero and turn axes off
Jun 10, 2020
d87feee
Prevent final frames from showing up in notebook if user stops render
Jun 10, 2020
5d41923
Allow render_scene_with_pointclouds_for_all_cameras to output frames …
Jun 12, 2020
247c484
Update indices in tutorial examples
Jun 12, 2020
8477c89
Style changes
Jun 12, 2020
50f605b
Clear memory in render_camera_channel_with_pointclouds if user stops …
Jun 12, 2020
9da4517
Graceful exit if users stops rendering
Jun 12, 2020
65c25f5
Change random seed for color scheme
Jun 12, 2020
ce775da
Style edits
Jun 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion python-sdk/nuscenes/lidarseg/lidarseg_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@ def plt_to_cv2(points: np.array, coloring: np.array, im, imsize: Tuple[int, int]
ax = plt.Axes(fig, [0., 0., 1., 1.])
fig.add_axes(ax)

ax.axis('off')
ax.margins(0, 0)

ax.imshow(im)
ax.scatter(points[0, :], points[1, :], c=coloring, s=5)

Expand Down Expand Up @@ -121,7 +124,7 @@ def get_colormap() -> np.ndarray:
return colormap


def get_arbitrary_colormap(num_classes: int, random_seed: int = 2020) -> np.ndarray:
def get_arbitrary_colormap(num_classes: int, random_seed: int = 93) -> np.ndarray:
"""
Create an arbitrary RGB colormap. Note that the RGB values are normalized between 0 and 1, not 0 and 255.
:param num_classes: Number of colors to create.
Expand Down
53 changes: 45 additions & 8 deletions python-sdk/nuscenes/nuscenes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1795,6 +1795,12 @@ def render_camera_channel_with_pointclouds(self, scene_token: str, camera_channe
key = cv2.waitKey()

if key == 27: # if ESC is pressed, exit.
plt.close('all') # To prevent figures from accumulating in memory.
# If rendering is stopped halfway, save whatever has been rendered so far into a video
# (if save_as_vid = True).
if save_as_vid:
out.write(mat)
out.release()
cv2.destroyAllWindows()
break

Expand Down Expand Up @@ -1827,7 +1833,10 @@ def render_scene_with_pointclouds_for_all_cameras(self, scene_token: str, out_pa
"""
Renders a full scene with all camera channels and the lidar segmentation labels for each camera.
:param scene_token: Unique identifier of scene to render.
:param out_path: Optional path to write a video file of the rendered frames.
:param out_path: Optional path to save the rendered figure to disk. The filename of each image will be
same as the original image's. If .avi is specified (e.g. '~/Desktop/my_rendered_scene.avi),
a video will be written instead of saving individual frames as images. Each image name wil
follow this format: <0-scene_number>_<frame_number>.jpg
:param filter_lidarseg_labels: Only show lidar points which belong to the given list of classes. If None
or the list is empty, all classes will be displayed.
:param freq: Display frequency (Hz).
Expand All @@ -1839,6 +1848,15 @@ def render_scene_with_pointclouds_for_all_cameras(self, scene_token: str, out_pa
"""
assert imsize[0] / imsize[1] == 16 / 9, "Aspect ratio should be 16/9."

if out_path is not None:
if os.path.splitext(out_path)[-1] == '.avi':
save_as_vid = True
else:
assert os.path.isdir(out_path), 'Error: {} does not exist.'.format(out_path)
save_as_vid = False
else:
save_as_vid = False

# Get records from DB.
scene_record = self.nusc.get('scene', scene_token)

Expand All @@ -1858,18 +1876,22 @@ def render_scene_with_pointclouds_for_all_cameras(self, scene_token: str, out_pa
'CAM_BACK_RIGHT': (2 * imsize[0], imsize[1]),
}

window_name = '{} {labels_type} (Space to pause, ESC to exit)'.format(
scene_record['name'], labels_type="(predictions)" if lidarseg_preds_folder else "")
cv2.namedWindow(window_name)
cv2.moveWindow(window_name, 0, 0)
horizontal_flip = ['CAM_BACK_LEFT', 'CAM_BACK', 'CAM_BACK_RIGHT'] # Flip these for aesthetic reasons.

if verbose:
window_name = '{} {labels_type} (Space to pause, ESC to exit)'.format(
scene_record['name'], labels_type="(predictions)" if lidarseg_preds_folder else "")
cv2.namedWindow(window_name)
cv2.moveWindow(window_name, 0, 0)
else:
window_name = None

slate = np.ones((2 * imsize[1], 3 * imsize[0], 3), np.uint8)
save_as_vid = False
if out_path is not None:

if save_as_vid:
assert os.path.splitext(out_path)[-1] == '.avi', 'Error: Video can only be saved in .avi format.'
fourcc = cv2.VideoWriter_fourcc(*'MJPG')
out = cv2.VideoWriter(out_path, fourcc, freq, slate.shape[1::-1])
save_as_vid = True
else:
out = None

Expand All @@ -1880,6 +1902,7 @@ def render_scene_with_pointclouds_for_all_cameras(self, scene_token: str, out_pa
keep_looping = False

sample_record = self.nusc.get('sample', current_token)
filename = '0' + scene_record['name'][5:] + '_{:02d}.jpg'.format(i)

for camera_channel in layout:
pointsensor_token = sample_record['data']['LIDAR_TOP']
Expand All @@ -1901,6 +1924,10 @@ def render_scene_with_pointclouds_for_all_cameras(self, scene_token: str, out_pa
if im is not None:
mat = plt_to_cv2(points, coloring, im, imsize)

if camera_channel in horizontal_flip:
# Flip image horizontally.
mat = cv2.flip(mat, 1)

slate[layout[camera_channel][1]: layout[camera_channel][1] + imsize[1],
layout[camera_channel][0]:layout[camera_channel][0] + imsize[0], :] = mat

Expand All @@ -1912,13 +1939,23 @@ def render_scene_with_pointclouds_for_all_cameras(self, scene_token: str, out_pa
key = cv2.waitKey()

if key == 27: # if ESC is pressed, exit.
plt.close('all') # To prevent figures from accumulating in memory.
# If rendering is stopped halfway, save whatever has been rendered so far into a video
# (if save_as_vid = True).
if save_as_vid:
out.write(slate)
out.release()
cv2.destroyAllWindows()
break

plt.close('all') # To prevent figures from accumulating in memory.

if save_as_vid:
out.write(slate)
elif out_path:
cv2.imwrite(os.path.join(out_path, filename), slate)
else:
pass

next_token = sample_record['next']
current_token = next_token
Expand Down
22 changes: 11 additions & 11 deletions python-sdk/tutorials/nuscenes_lidarseg_tutorial.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@
"nusc.render_sample_data(sample_data_token,\n",
" with_anns=False,\n",
" show_lidarseg_labels=True,\n",
" filter_lidarseg_labels=[3, 4, 5])"
" filter_lidarseg_labels=[9, 14, 18])"
]
},
{
Expand Down Expand Up @@ -184,7 +184,7 @@
" camera_channel='CAM_BACK',\n",
" render_intensity=False,\n",
" show_lidarseg_labels=True,\n",
" filter_lidarseg_labels=[3, 4, 5],\n",
" filter_lidarseg_labels=[9, 14, 18],\n",
" show_lidarseg_legend=True,\n",
" verbose=True)"
]
Expand All @@ -207,7 +207,7 @@
" camera_channel='CAM_BACK',\n",
" render_intensity=False,\n",
" show_lidarseg_labels=True,\n",
" filter_lidarseg_labels=[30, 15],\n",
" filter_lidarseg_labels=[21, 11],\n",
" show_lidarseg_legend=True,\n",
" verbose=True,\n",
" render_if_no_points=False)"
Expand Down Expand Up @@ -236,7 +236,7 @@
"source": [
"nusc.render_sample(my_sample['token'],\n",
" show_lidarseg_labels=True,\n",
" filter_lidarseg_labels=[3, 4, 5],\n",
" filter_lidarseg_labels=[9, 14, 18],\n",
" verbose=True)"
]
},
Expand Down Expand Up @@ -282,7 +282,7 @@
"# import os\n",
"# nusc.render_camera_channel_with_pointclouds(my_scene['token'], \n",
"# 'CAM_BACK', \n",
"# filter_lidarseg_labels=[6, 36],\n",
"# filter_lidarseg_labels=[15, 30],\n",
"# render_if_no_points=True, \n",
"# verbose=True, \n",
"# imsize=(1280, 720))"
Expand All @@ -304,7 +304,7 @@
"outputs": [],
"source": [
"# nusc.render_camera_channel_with_pointclouds(my_scene['token'], 'CAM_BACK',\n",
"# filter_lidarseg_labels=[6, 36],\n",
"# filter_lidarseg_labels=[15, 30],\n",
"# render_if_no_points=True,\n",
"# verbose=True,\n",
"# imsize=(1280, 720),\n",
Expand All @@ -327,7 +327,7 @@
"outputs": [],
"source": [
"# nusc.render_camera_channel_with_pointclouds(my_scene['token'], 'CAM_BACK',\n",
"# filter_lidarseg_labels=[6],\n",
"# filter_lidarseg_labels=[15],\n",
"# render_if_no_points=True,\n",
"# verbose=True,\n",
"# imsize=(1280, 720),\n",
Expand All @@ -351,7 +351,7 @@
"outputs": [],
"source": [
"# nusc.render_scene_with_pointclouds_for_all_cameras(my_scene['token'], \n",
"# filter_lidarseg_labels=[32, 1],\n",
"# filter_lidarseg_labels=[26, 9],\n",
"# out_path=os.path.expanduser('~/Desktop/my_rendered_scene.avi'))"
]
},
Expand Down Expand Up @@ -388,7 +388,7 @@
" camera_channel='CAM_BACK',\n",
" render_intensity=False,\n",
" show_lidarseg_labels=True,\n",
" filter_lidarseg_labels=[3, 4, 5],\n",
" filter_lidarseg_labels=[9, 14, 18],\n",
" show_lidarseg_legend=True,\n",
" verbose=True,\n",
" lidarseg_preds_bin_path=my_predictions_bin_file)"
Expand Down Expand Up @@ -418,7 +418,7 @@
"\n",
"# nusc.render_camera_channel_with_pointclouds(my_scene['token'], \n",
"# 'CAM_BACK', \n",
"# filter_lidarseg_labels=[6, 36],\n",
"# filter_lidarseg_labels=[15, 30],\n",
"# render_if_no_points=True, \n",
"# verbose=True, \n",
"# imsize=(1280, 720),\n",
Expand Down Expand Up @@ -450,7 +450,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.7"
"version": "3.7.6"
}
},
"nbformat": 4,
Expand Down