From 6ee3c6af9a02170b2f02a1a7031b4d95883e9707 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Mon, 13 Nov 2023 15:47:07 +0100 Subject: [PATCH] Fix line & points (& depth clouds points) radii being unaffected by scale & projection via Pinhole (#4199) ### What Longstanding issue! * Fixes #1223 * Fixes #1219 * Fixes #2494 * Replaces #4196 Shader only fix, in the future the scale factor shouldn't be extracted on the fly for every vertex out of the transform and instead passed in, but I wanted to keep the change minimal. The added vertex shading cost is unlikely to matter all that much _short term_. (also was very nice iterating on this and get before/after screenshots ;)) Throw-away test script for this: ```py import numpy as np import rerun as rr rr.init("scale fix test!!!", spawn=True) ############################# # #2494 & #1219 ############################# rr.log("world/camera", rr.ViewCoordinates.RDF, timeless=True) rr.log( "world/camera/image", rr.Pinhole( image_from_camera=np.array([[500, 0, 250], [0, 500, 250], [0, 0, 1]]), width=500, height=500, ), ) rr.log("world/camera/image/rgb", rr.Image(np.ones((500, 500, 3)))) rr.log( "world/camera/image/points", rr.Points2D(np.random.uniform(0, 500, (30, 2)), radii=1), ) ############################# # #1223 ############################# rr.log( "scaling_stuff/points_unscaled", rr.Points3D( np.random.uniform(0, 1, (30, 3)), radii=0.1, ), ) rr.log( "scaling_stuff/points_scaled", rr.Points3D( np.random.uniform(0, 1, (30, 3)), radii=0.1, ), rr.Transform3D(scale=2.0, translation=[2, 2, 2]), ) rr.log( "scaling_stuff/lines_unscaled", rr.LineStrips3D([[0, 1, 0], [0, 1, 1], [0, 0, 3]], radii=0.1), ) rr.log( "scaling_stuff/lines_scaled", rr.LineStrips3D([[0, 1, 0], [0, 1, 1], [0, 0, 3]], radii=0.1), rr.Transform3D(scale=2.0, translation=[2, 2, 2]), ) ``` Result: Before: ![image](https://github.com/rerun-io/rerun/assets/1220815/29d8f98a-c2f9-4503-84c7-83fc7a8814ff) After: ![image](https://github.com/rerun-io/rerun/assets/1220815/3d555ad9-790f-446f-a3c0-d72904db7a84) This not only fixes issues with 2D->3D but also with 3D->2D. Here we add the 3D points to the 2D camera and set a world space size for the points: Before: ![image](https://github.com/rerun-io/rerun/assets/1220815/038c6826-d6ee-4e00-93f4-2a964bc56abe) After: ![image](https://github.com/rerun-io/rerun/assets/1220815/132ba0d6-9906-4f22-b4f6-c6ced82e3748) ### Checklist * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested [demo.rerun.io](https://demo.rerun.io/pr/4199) (if applicable) * [x] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG - [PR Build Summary](https://build.rerun.io/pr/4199) - [Docs preview](https://rerun.io/preview/7465b1650dd9be5c712c994827ced405c142edad/docs) - [Examples preview](https://rerun.io/preview/7465b1650dd9be5c712c994827ced405c142edad/examples) - [Recent benchmark results](https://ref.rerun.io/dev/bench/) - [Wasm size tracking](https://ref.rerun.io/dev/sizes/) --- crates/re_renderer/shader/depth_cloud.wgsl | 8 +++++-- crates/re_renderer/shader/lines.wgsl | 3 ++- crates/re_renderer/shader/point_cloud.wgsl | 9 ++++++-- crates/re_renderer/shader/utils/size.wgsl | 21 +++++++++++++++++-- .../re_renderer/shader/utils/sphere_quad.wgsl | 17 ++++++--------- 5 files changed, 40 insertions(+), 18 deletions(-) diff --git a/crates/re_renderer/shader/depth_cloud.wgsl b/crates/re_renderer/shader/depth_cloud.wgsl index 6b79215c0234..e0c966754116 100644 --- a/crates/re_renderer/shader/depth_cloud.wgsl +++ b/crates/re_renderer/shader/depth_cloud.wgsl @@ -161,8 +161,12 @@ fn vs_main(@builtin(vertex_index) vertex_idx: u32) -> VertexOut { if 0.0 < point_data.unresolved_radius { // Span quad - let quad = sphere_or_circle_quad_span(vertex_idx, point_data.pos_in_world, point_data.unresolved_radius, - depth_cloud_info.radius_boost_in_ui_points, false); + let camera_distance = distance(frame.camera_position, point_data.pos_in_world); + let world_scale_factor = average_scale_from_transform(depth_cloud_info.world_from_rdf); // TODO(andreas): somewhat costly, should precompute this + let world_radius = unresolved_size_to_world(point_data.unresolved_radius, camera_distance, + frame.auto_size_points, world_scale_factor) + + world_size_from_point_size(depth_cloud_info.radius_boost_in_ui_points, camera_distance); + let quad = sphere_or_circle_quad_span(vertex_idx, point_data.pos_in_world, world_radius, false); out.pos_in_clip = frame.projection_from_world * vec4f(quad.pos_in_world, 1.0); out.pos_in_world = quad.pos_in_world; out.point_radius = quad.point_resolved_radius; diff --git a/crates/re_renderer/shader/lines.wgsl b/crates/re_renderer/shader/lines.wgsl index 8117a209d51f..bd14f2fe1fac 100644 --- a/crates/re_renderer/shader/lines.wgsl +++ b/crates/re_renderer/shader/lines.wgsl @@ -212,7 +212,8 @@ fn vs_main(@builtin(vertex_index) vertex_idx: u32) -> VertexOut { camera_ray = camera_ray_to_world_pos_perspective(center_position); } let camera_distance = distance(camera_ray.origin, center_position); - var strip_radius = unresolved_size_to_world(strip_data.unresolved_radius, camera_distance, frame.auto_size_lines); + let world_scale_factor = average_scale_from_transform(batch.world_from_obj); // TODO(andreas): somewhat costly, should precompute this + var strip_radius = unresolved_size_to_world(strip_data.unresolved_radius, camera_distance, frame.auto_size_lines, world_scale_factor); // If the triangle cap is longer than the quad would be otherwise, we need to stunt it, otherwise we'd get artifacts. var triangle_cap_length = batch.triangle_cap_length_factor * strip_radius; diff --git a/crates/re_renderer/shader/point_cloud.wgsl b/crates/re_renderer/shader/point_cloud.wgsl index 7a8d3abd3253..a1c37db4e447 100644 --- a/crates/re_renderer/shader/point_cloud.wgsl +++ b/crates/re_renderer/shader/point_cloud.wgsl @@ -98,8 +98,13 @@ fn vs_main(@builtin(vertex_index) vertex_idx: u32) -> VertexOut { let point_data = read_data(quad_idx); // Span quad - let quad = sphere_or_circle_quad_span(vertex_idx, point_data.pos, point_data.unresolved_radius, - draw_data.radius_boost_in_ui_points, has_any_flag(batch.flags, FLAG_DRAW_AS_CIRCLES)); + let camera_distance = distance(frame.camera_position, point_data.pos); + let world_scale_factor = average_scale_from_transform(batch.world_from_obj); // TODO(andreas): somewhat costly, should precompute this + let world_radius = unresolved_size_to_world(point_data.unresolved_radius, camera_distance, + frame.auto_size_points, world_scale_factor) + + world_size_from_point_size(draw_data.radius_boost_in_ui_points, camera_distance); + let quad = sphere_or_circle_quad_span(vertex_idx, point_data.pos, world_radius, + has_any_flag(batch.flags, FLAG_DRAW_AS_CIRCLES)); // Output, transform to projection space and done. var out: VertexOut; diff --git a/crates/re_renderer/shader/utils/size.wgsl b/crates/re_renderer/shader/utils/size.wgsl index afd1cf6300b3..8d0ad8c2cd2e 100644 --- a/crates/re_renderer/shader/utils/size.wgsl +++ b/crates/re_renderer/shader/utils/size.wgsl @@ -7,7 +7,12 @@ fn world_size_from_point_size(size_in_points: f32, camera_distance: f32) -> f32 return approx_pixel_world_size_at(camera_distance) * pixel_size; } -fn unresolved_size_to_world(_unresolved_size: f32, camera_distance: f32, auto_size: f32) -> f32 { +// Resolves a size (see size.rs!) to a world scale size. +// +// world_size_scale: +// Scale factor that is applied iff the size is a world size. +// This is usually part of your object->world transform. +fn unresolved_size_to_world(_unresolved_size: f32, camera_distance: f32, auto_size: f32, world_size_scale: f32) -> f32 { // Resolve auto size. var unresolved_size: f32; if _unresolved_size >= f32max { @@ -23,9 +28,21 @@ fn unresolved_size_to_world(_unresolved_size: f32, camera_distance: f32, auto_si // Is it a world size? if unresolved_size > 0.0 { - return unresolved_size; + return unresolved_size * world_size_scale; } // Negative size indicates size in points. return world_size_from_point_size(-unresolved_size, camera_distance); } + +// Determines the scale factor of a matrix +// +// This quite expensive, you may want to precompute this. +fn average_scale_from_transform(transform: mat4x4f) -> f32 { + // Source: https://math.stackexchange.com/a/1463487 + // Won't work with negative scale. + // Note we're only look at the scale, not at shear + let scale = vec3f(length(transform[0].xyz), length(transform[1].xyz), length(transform[2].xyz)); + // Get geometric mean + return pow(scale.x * scale.y * scale.z, 0.3333); +} diff --git a/crates/re_renderer/shader/utils/sphere_quad.wgsl b/crates/re_renderer/shader/utils/sphere_quad.wgsl index 3800a15b272c..ad61de568e9e 100644 --- a/crates/re_renderer/shader/utils/sphere_quad.wgsl +++ b/crates/re_renderer/shader/utils/sphere_quad.wgsl @@ -69,14 +69,7 @@ struct SphereQuadData { /// Span a quad onto which circles or perspective correct spheres can be drawn. /// /// Note that in orthographic mode, spheres are always circles. -fn sphere_or_circle_quad_span(vertex_idx: u32, point_pos: vec3f, point_unresolved_radius: f32, - radius_boost_in_ui_points: f32, force_circle: bool) -> SphereQuadData { - // Resolve radius to a world size. We need the camera distance for this, which is useful later on. - let to_camera = frame.camera_position - point_pos; - let camera_distance = length(to_camera); - let radius = unresolved_size_to_world(point_unresolved_radius, camera_distance, frame.auto_size_points) + - world_size_from_point_size(radius_boost_in_ui_points, camera_distance); - +fn sphere_or_circle_quad_span(vertex_idx: u32, point_pos: vec3f, world_radius: f32, force_circle: bool) -> SphereQuadData { // Basic properties of the vertex we're at. let local_idx = vertex_idx % 6u; let top_bottom = f32(local_idx <= 1u || local_idx == 5u) * 2.0 - 1.0; // 1 for a top vertex, -1 for a bottom vertex. @@ -85,12 +78,14 @@ fn sphere_or_circle_quad_span(vertex_idx: u32, point_pos: vec3f, point_unresolve // Span quad var pos: vec3f; if is_camera_orthographic() || force_circle { - pos = circle_quad(point_pos, radius, top_bottom, left_right); + pos = circle_quad(point_pos, world_radius, top_bottom, left_right); } else { - pos = sphere_quad(point_pos, radius, top_bottom, left_right, to_camera, camera_distance); + let to_camera = frame.camera_position - point_pos; + let camera_distance = length(to_camera); + pos = sphere_quad(point_pos, world_radius, top_bottom, left_right, to_camera, camera_distance); } - return SphereQuadData(pos, radius); + return SphereQuadData(pos, world_radius); } /// Computes coverage of a 3D sphere placed at `sphere_center` in the fragment shader using the currently set camera.