From f1a0f3edd6b6a7d96352f43f83844111e8e9ea35 Mon Sep 17 00:00:00 2001 From: Saurabh Khanduja Date: Tue, 26 Sep 2023 18:25:00 +0200 Subject: [PATCH] Improve Documentation - Fix code links and improve tutorial layout (#6321) * Correct code tagging in interactive_visualization tutorial. * Highlight the methods that are being introduced at the start. * Non-Blocking VIZ - Add add_geometry step which is not optional. * Non-Blocking VIZ - Split main code to simpler function and fix code blocks in docs. * Fix Typos * Update docs with new argument values. * Fix grammar and dedent for overly indented code * Integrate_Custom - Optimize imports, add early return, use f-strings and remove unnecessary parentheses. * Fix default value and remove BooleanOptionalAction, its only available >= python 3.9 * Disable use of cuda for tensor based reconstruction * Customized Integration - Fix code lines * Ray Casting - Fix code lines * Dense Slam - Fix code lines and slight grammar. * Fix some f-strings and extra parantheses * Make Fragments - Fix code lines * Code for batch processing was removed long back in #521 in 2018 * Register Fragments - Fix code docs * Register fragments - Fix code lines * Integrate Scene - Fix code lines * Change layout of tutorials - only done at top level. * Add new lines before bullet/numbered list, else list rendering fails. * Bugfix - Fix code lines for Integration and add filename as done in v0.14.0 * Remove python version 3.6 as supported versions by open3d. * Fix line number using lineno-match directive. * Fix some typos detected automatically. * Replace prepend with lineno-match * Correct filename documented in the files * Fix lineno-start. * Fix PR comments * Revert "Code for batch processing was removed long back in #521 in 2018" This reverts commit d65fb67a17d6a7d1f8e3a55effea7bc53a9bf849. * Correct batch processing section documentation and native multiprocessing. * Use native multiprocessing for register fragments and refinement of registration --------- Co-authored-by: Sameer Sheorey <41028320+ssheorey@users.noreply.github.com> --- README.md | 2 +- cpp/open3d/core/AdvancedIndexing.cpp | 2 +- cpp/open3d/core/AdvancedIndexing.h | 4 +- cpp/open3d/geometry/BoundingVolume.h | 2 +- cpp/open3d/geometry/VoxelGrid.h | 2 +- .../ContinuousConvBackpropFilter.cuh | 2 +- .../misc/BuildSpatialHashTableOps.cpp | 2 +- cpp/open3d/t/geometry/BoundingVolume.cpp | 2 +- cpp/open3d/t/geometry/BoundingVolume.h | 4 +- cpp/open3d/visualization/gui/Window.cpp | 2 +- docs/index.rst | 6 +- docs/jupyter/core/hashmap.ipynb | 4 + .../reconstruction_system/integrate_scene.rst | 5 +- .../reconstruction_system/make_fragments.rst | 29 ++- .../refine_registration.rst | 19 +- .../register_fragments.rst | 24 +- .../reconstruction_system/system_overview.rst | 6 +- .../customized_integration.rst | 16 +- .../t_reconstruction_system/dense_slam.rst | 10 +- .../t_reconstruction_system/index.rst | 6 +- .../t_reconstruction_system/integration.rst | 26 +- .../t_reconstruction_system/ray_casting.rst | 14 +- .../voxel_block_grid.rst | 18 +- docs/tutorial/visualization/cpu_rendering.rst | 2 +- .../interactive_visualization.rst | 36 +-- .../non_blocking_visualization.rst | 17 +- .../debug/visualize_fragments.py | 2 +- .../reconstruction_system/make_fragments.py | 16 +- .../refine_registration.py | 38 +-- .../register_fragments.py | 31 +-- .../sensors/realsense_helper.py | 2 +- .../python/t_reconstruction_system/common.py | 6 +- .../python/t_reconstruction_system/config.py | 15 +- .../t_reconstruction_system/dense_slam.py | 4 +- .../t_reconstruction_system/dense_slam_gui.py | 4 +- .../integrate_custom.py | 222 +++++++++--------- .../pose_graph_optim.py | 4 +- .../t_reconstruction_system/ray_casting.py | 2 +- .../interactive_visualization.py | 43 ++-- .../non_blocking_visualization.py | 18 +- 40 files changed, 359 insertions(+), 310 deletions(-) diff --git a/README.md b/README.md index b3d713a42d2..d3c583ee94f 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ For more, please visit the [Open3D documentation](http://www.open3d.org/docs). ## Python quick start Pre-built pip packages support Ubuntu 18.04+, macOS 10.15+ and Windows 10+ -(64-bit) with Python 3.6-3.10. +(64-bit) with Python 3.7-3.10. ```bash # Install diff --git a/cpp/open3d/core/AdvancedIndexing.cpp b/cpp/open3d/core/AdvancedIndexing.cpp index f2bbf306c3f..86ce53148ba 100644 --- a/cpp/open3d/core/AdvancedIndexing.cpp +++ b/cpp/open3d/core/AdvancedIndexing.cpp @@ -206,7 +206,7 @@ void AdvancedIndexPreprocessor::RunPreprocess() { // If the indexed_shape_ contains a dimension of size 0 but the // replacement shape does not, the index is out of bounds. This is because // there is no valid number to index an empty tensor. - // Normally, out of bounds is detected in the advanded indexing kernel. We + // Normally, out of bounds is detected in the advanced indexing kernel. We // detected here for more helpful error message. auto contains_zero = [](const SizeVector& vals) -> bool { return std::any_of(vals.begin(), vals.end(), diff --git a/cpp/open3d/core/AdvancedIndexing.h b/cpp/open3d/core/AdvancedIndexing.h index b01d922a61d..17a8253085a 100644 --- a/cpp/open3d/core/AdvancedIndexing.h +++ b/cpp/open3d/core/AdvancedIndexing.h @@ -49,7 +49,7 @@ class AdvancedIndexPreprocessor { const Tensor& tensor, const std::vector& index_tensors); /// Expand all tensors to the broadcasted shape, 0-dim tensors are ignored. - /// Thorws exception if the common broadcasted shape does not exist. + /// Throws exception if the common broadcasted shape does not exist. static std::pair, SizeVector> ExpandToCommonShapeExceptZeroDim(const std::vector& index_tensors); @@ -127,7 +127,7 @@ class AdvancedIndexer { if (indexed_shape.size() != indexed_strides.size()) { utility::LogError( "Internal error: indexed_shape's ndim {} does not equal to " - "indexd_strides' ndim {}", + "indexed_strides' ndim {}", indexed_shape.size(), indexed_strides.size()); } num_indices_ = indexed_shape.size(); diff --git a/cpp/open3d/geometry/BoundingVolume.h b/cpp/open3d/geometry/BoundingVolume.h index 11586a2132d..e808b232612 100644 --- a/cpp/open3d/geometry/BoundingVolume.h +++ b/cpp/open3d/geometry/BoundingVolume.h @@ -21,7 +21,7 @@ class AxisAlignedBoundingBox; /// \brief A bounding box oriented along an arbitrary frame of reference. /// /// The oriented bounding box is defined by its center position, rotation -/// maxtrix and extent. +/// matrix and extent. class OrientedBoundingBox : public Geometry3D { public: /// \brief Default constructor. diff --git a/cpp/open3d/geometry/VoxelGrid.h b/cpp/open3d/geometry/VoxelGrid.h index b0d8cc458ed..30a228206e0 100644 --- a/cpp/open3d/geometry/VoxelGrid.h +++ b/cpp/open3d/geometry/VoxelGrid.h @@ -245,7 +245,7 @@ class VoxelGrid : public Geometry3D { public: /// Size of the voxel. double voxel_size_ = 0.0; - /// Coorindate of the origin point. + /// Coordinate of the origin point. Eigen::Vector3d origin_ = Eigen::Vector3d::Zero(); /// Voxels contained in voxel grid std::unordered_map ¢er = utility::nullopt); /// \brief Add operation for axis-aligned bounding box. - /// The device of ohter box must be the same as the device of the current + /// The device of other box must be the same as the device of the current /// box. AxisAlignedBoundingBox &operator+=(const AxisAlignedBoundingBox &other); @@ -223,7 +223,7 @@ class AxisAlignedBoundingBox : public Geometry, public DrawableGeometry { /// \brief A bounding box oriented along an arbitrary frame of reference. /// /// - (center, rotation, extent): The oriented bounding box is defined by its -/// center position, rotation maxtrix and extent. +/// center position, rotation matrix and extent. /// - Usage /// - OrientedBoundingBox::GetCenter() /// - OrientedBoundingBox::SetCenter(const core::Tensor ¢er) diff --git a/cpp/open3d/visualization/gui/Window.cpp b/cpp/open3d/visualization/gui/Window.cpp index 0ee562b4310..bb023903792 100644 --- a/cpp/open3d/visualization/gui/Window.cpp +++ b/cpp/open3d/visualization/gui/Window.cpp @@ -707,7 +707,7 @@ Widget::DrawResult DrawChild(DrawContext& dc, const char* name, std::shared_ptr child, Mode mode) { - // Note: ImGUI's concept of a "window" is really a moveable child of the + // Note: ImGUI's concept of a "window" is really a movable child of the // OS window. We want a child to act like a child of the OS window, // like native UI toolkits, Qt, etc. So the top-level widgets of // a window are drawn using ImGui windows whose frame is specified diff --git a/docs/index.rst b/docs/index.rst index ca0da37702d..083cd7e23fa 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -34,13 +34,13 @@ Open3D: A Modern Library for 3D Data Processing :maxdepth: 2 :caption: Tutorial + tutorial/core/index tutorial/geometry/index tutorial/t_geometry/index + tutorial/data/index + tutorial/visualization/index tutorial/pipelines/index tutorial/t_pipelines/index - tutorial/visualization/index - tutorial/core/index - tutorial/data/index tutorial/reconstruction_system/index tutorial/t_reconstruction_system/index tutorial/sensor/index diff --git a/docs/jupyter/core/hashmap.ipynb b/docs/jupyter/core/hashmap.ipynb index 31f57aace92..6d05755c4f9 100644 --- a/docs/jupyter/core/hashmap.ipynb +++ b/docs/jupyter/core/hashmap.ipynb @@ -88,19 +88,23 @@ "Next we show how to insert a batch of (key, value) pairs. You'll need to prepare two tensors:\n", "\n", "The `keys` tensor contains all keys. \n", + "\n", "- The `keys` tensor must be on the same device as the hash map. \n", "- The shape of the `keys` tensor is `key_elment_shape` with `N` prefixed to the front. \n", "\n", "For example \n", + " \n", "1. if `key_element_shape == ()`, `keys.shape == (N,)`; \n", "2. if `key_element_shape == (3,)`, `keys.shape == (N, 3).`; \n", "3. if `key_element_shape == (8, 8, 8)`, `keys.shape == (N, 8, 8, 8).`\n", " \n", "The `vals` tensor contains all values. \n", + " \n", "- The `vals` tensor must be on the same device as the hash map. \n", "- The shape of the `vals` tensor is `val_elment_shape` with `N` prefixed to the front. \n", "\n", "For example \n", + "\n", "1. if `val_elment_shape == ()`, `vals.shape == (N,)`; \n", "2. if `val_elment_shape == (3,)`, `vals.shape == (N, 3).`;\n", "3. if `val_elment_shape == (8, 8, 8)`, `vals.shape == (N, 8, 8, 8).`" diff --git a/docs/tutorial/reconstruction_system/integrate_scene.rst b/docs/tutorial/reconstruction_system/integrate_scene.rst index 4a56b188ab3..d480d782219 100644 --- a/docs/tutorial/reconstruction_system/integrate_scene.rst +++ b/docs/tutorial/reconstruction_system/integrate_scene.rst @@ -21,9 +21,10 @@ Integrate RGBD frames .. literalinclude:: ../../../examples/python/reconstruction_system/integrate_scene.py :language: python - :lineno-start: 38 - :lines: 27,40-72 + :pyobject: scalable_integrate_rgb_frames + :end-at: o3d.visualization.draw_geometries([mesh]) :linenos: + :lineno-match: This function first reads the alignment results from both :ref:`reconstruction_system_make_fragments` and diff --git a/docs/tutorial/reconstruction_system/make_fragments.rst b/docs/tutorial/reconstruction_system/make_fragments.rst index 2485bfad2b3..d3f53b0575f 100644 --- a/docs/tutorial/reconstruction_system/make_fragments.rst +++ b/docs/tutorial/reconstruction_system/make_fragments.rst @@ -25,9 +25,9 @@ Register RGBD image pairs .. literalinclude:: ../../../examples/python/reconstruction_system/make_fragments.py :language: python - :lineno-start: 46 - :lines: 27,47-76 + :pyobject: register_one_rgbd_pair :linenos: + :lineno-match: The function reads a pair of RGBD images and registers the ``source_rgbd_image`` to the ``target_rgbd_image``. The Open3D function ``compute_rgbd_odometry`` is @@ -45,9 +45,9 @@ Multiway registration .. literalinclude:: ../../../examples/python/reconstruction_system/make_fragments.py :language: python - :lineno-start: 76 - :lines: 27,77-123 + :pyobject: make_posegraph_for_fragment :linenos: + :lineno-match: This script uses the technique demonstrated in :ref:`/tutorial/pipelines/multiway_registration.ipynb`. The function @@ -61,9 +61,9 @@ function ``optimize_posegraph_for_fragment``. .. literalinclude:: ../../../examples/python/reconstruction_system/optimize_posegraph.py :language: python - :lineno-start: 51 - :lines: 27,52-63 + :pyobject: optimize_posegraph_for_fragment :linenos: + :lineno-match: This function calls ``global_optimization`` to estimate poses of the RGBD images. @@ -74,11 +74,11 @@ Make a fragment .. literalinclude:: ../../../examples/python/reconstruction_system/make_fragments.py :language: python - :lineno-start: 124 - :lines: 27,125-146 + :pyobject: integrate_rgb_frames_for_fragment :linenos: + :lineno-match: -Once the poses are estimates, :ref:`/tutorial/pipelines/rgbd_integration.ipynb` +Once the poses are estimated, :ref:`/tutorial/pipelines/rgbd_integration.ipynb` is used to reconstruct a colored fragment from each RGBD sequence. Batch processing @@ -86,11 +86,16 @@ Batch processing .. literalinclude:: ../../../examples/python/reconstruction_system/make_fragments.py :language: python - :lineno-start: 181 - :lines: 27,182-205 + :start-at: def process_single_fragment(fragment_id, color_files, depth_files, n_files, :linenos: + :lineno-match: + +The ``process_single_fragment`` function calls each individual function explained above. +The ``run`` function determines the number of fragments to generate based on the number +of images in the dataset and the configuration value ``n_frames_per_fragment``. +Subsequently, it invokes ``process_single_fragment`` for each of these fragments. +Furthermore, it leverages multiprocessing to speed up computation of all fragments. -The main function calls each individual function explained above. .. _reconstruction_system_make_fragments_results: diff --git a/docs/tutorial/reconstruction_system/refine_registration.rst b/docs/tutorial/reconstruction_system/refine_registration.rst index 2afcb24b6bc..e512255f671 100644 --- a/docs/tutorial/reconstruction_system/refine_registration.rst +++ b/docs/tutorial/reconstruction_system/refine_registration.rst @@ -20,9 +20,9 @@ Fine-grained registration .. literalinclude:: ../../../examples/python/reconstruction_system/refine_registration.py :language: python - :lineno-start: 63 - :lines: 27,64-136 :linenos: + :pyobject: multiscale_icp + :lineno-match: Two options are given for the fine-grained registration. The ``color`` option is recommended since it uses color information to prevent drift. See [Park2017]_ @@ -33,9 +33,9 @@ Multiway registration .. literalinclude:: ../../../examples/python/reconstruction_system/refine_registration.py :language: python - :lineno-start: 40 - :lines: 27,41-63 :linenos: + :pyobject: update_posegraph_for_scene + :lineno-match: This script uses the technique demonstrated in :ref:`/tutorial/pipelines/multiway_registration.ipynb`. Function ``update_posegraph_for_scene`` builds a pose graph for multiway registration of all fragments. Each graph node represents a fragment and its pose which transforms the geometry to the global space. @@ -44,21 +44,20 @@ for multiway registration. .. literalinclude:: ../../../examples/python/reconstruction_system/optimize_posegraph.py :language: python - :lineno-start: 63 - :lines: 27,64-73 :linenos: + :pyobject: optimize_posegraph_for_scene + :lineno-match: Main registration loop `````````````````````````````````````` -The function ``make_posegraph_for_refined_scene`` below calls all the functions - introduced above. +The function ``make_posegraph_for_refined_scene`` below calls all the functions introduced above. .. literalinclude:: ../../../examples/python/reconstruction_system/refine_registration.py :language: python - :lineno-start: 173 - :lines: 27,174-223 :linenos: + :pyobject: make_posegraph_for_refined_scene + :lineno-match: The main workflow is: pairwise local refinement -> multiway registration. diff --git a/docs/tutorial/reconstruction_system/register_fragments.rst b/docs/tutorial/reconstruction_system/register_fragments.rst index 7e3486589a8..b86adc62fda 100644 --- a/docs/tutorial/reconstruction_system/register_fragments.rst +++ b/docs/tutorial/reconstruction_system/register_fragments.rst @@ -20,9 +20,9 @@ Preprocess point cloud .. literalinclude:: ../../../examples/python/reconstruction_system/register_fragments.py :language: python - :lineno-start: 41 - :lines: 27,42-54 + :pyobject: preprocess_point_cloud :linenos: + :lineno-match: This function downsamples a point cloud to make it sparser and regularly distributed. Normals and FPFH feature are precomputed. See @@ -36,9 +36,9 @@ Compute initial registration .. literalinclude:: ../../../examples/python/reconstruction_system/register_fragments.py :language: python - :lineno-start: 85 - :lines: 27,86-114 + :pyobject: compute_initial_registration :linenos: + :lineno-match: This function computes a rough alignment between two fragments. If the fragments are neighboring fragments, the rough alignment is determined by an aggregating @@ -53,9 +53,9 @@ Pairwise global registration .. literalinclude:: ../../../examples/python/reconstruction_system/register_fragments.py :language: python - :lineno-start: 54 - :lines: 27,55-85 + :pyobject: register_point_cloud_fpfh :linenos: + :lineno-match: This function uses :ref:`/tutorial/pipelines/global_registration.ipynb#RANSAC` or :ref:`/tutorial/pipelines/global_registration.ipynb#fast-global-registration` for pairwise global registration. @@ -66,9 +66,9 @@ Multiway registration .. literalinclude:: ../../../examples/python/reconstruction_system/register_fragments.py :language: python - :lineno-start: 114 - :lines: 27,115-137 + :pyobject: update_posegraph_for_scene :linenos: + :lineno-match: This script uses the technique demonstrated in :ref:`/tutorial/pipelines/multiway_registration.ipynb`. The function @@ -81,9 +81,9 @@ called for multiway registration. .. literalinclude:: ../../../examples/python/reconstruction_system/optimize_posegraph.py :language: python - :lineno-start: 63 - :lines: 27,64-73 + :pyobject: optimize_posegraph_for_scene :linenos: + :lineno-match: Main registration loop `````````````````````````````````````` @@ -94,9 +94,9 @@ multiway registration. .. literalinclude:: ../../../examples/python/reconstruction_system/register_fragments.py :language: python - :lineno-start: 167 - :lines: 27,168-210 + :pyobject: make_posegraph_for_scene :linenos: + :lineno-match: Results `````````````````````````````````````` diff --git a/docs/tutorial/reconstruction_system/system_overview.rst b/docs/tutorial/reconstruction_system/system_overview.rst index 2cae7ed387e..0394d629c1e 100644 --- a/docs/tutorial/reconstruction_system/system_overview.rst +++ b/docs/tutorial/reconstruction_system/system_overview.rst @@ -48,7 +48,7 @@ Getting the example code .. code-block:: sh - # Activate your conda enviroment, where you have installed open3d pip package. + # Activate your conda environment, where you have installed open3d pip package. # Clone the Open3D github repository and go to the example. cd examples/python/reconstruction_system/ @@ -69,7 +69,7 @@ Running the example with default dataset. python run_system.py --make --register --refine --integrate Changing the default dataset. -One may change the default dataset to other avaialble datasets. +One may change the default dataset to other available datasets. Currently the following datasets are available: 1. Lounge (keyword: ``lounge``) (Default) @@ -81,7 +81,7 @@ Currently the following datasets are available: .. code-block:: sh - # Using jack_jack as the default dataset. + # Using bedroom as the default dataset. python run_system.py --default_dataset 'bedroom' --make --register --refine --integrate Running the example with custom dataset using config file. diff --git a/docs/tutorial/t_reconstruction_system/customized_integration.rst b/docs/tutorial/t_reconstruction_system/customized_integration.rst index 79625fd5ec2..78e252fa8d5 100644 --- a/docs/tutorial/t_reconstruction_system/customized_integration.rst +++ b/docs/tutorial/t_reconstruction_system/customized_integration.rst @@ -10,8 +10,8 @@ The frustum **block** selection remains the same, but then we manually activate .. literalinclude:: ../../../examples/python/t_reconstruction_system/integrate_custom.py :language: python - :lineno-start: 78 - :lines: 27,79-87 + :lineno-start: 60 + :lines: 8,61-68 Voxel Indices `````````````` @@ -19,8 +19,8 @@ We can then unroll **voxel** indices in these **blocks** into a flattened array, .. literalinclude:: ../../../examples/python/t_reconstruction_system/integrate_custom.py :language: python - :lineno-start: 91 - :lines: 27,92-93 + :lineno-start: 72 + :lines: 8,73-74 Up to now we have finished preparation. Then we can perform customized geometry transformation in the Tensor interface, with the same fashion as we conduct in numpy or pytorch. @@ -30,8 +30,8 @@ We first transform the voxel coordinates to the frame's coordinate system, proje .. literalinclude:: ../../../examples/python/t_reconstruction_system/integrate_custom.py :language: python - :lineno-start: 99 - :lines: 27,100-118 + :lineno-start: 80 + :lines: 8,81-98 Customized integration ```````````````````````` @@ -43,7 +43,7 @@ With the data association, we are able to conduct integration. In this example, .. literalinclude:: ../../../examples/python/t_reconstruction_system/integrate_custom.py :language: python - :lineno-start: 118 - :lines: 27,119-128,133-151 + :lineno-start: 98 + :lines: 8,99-108,113-132 You may follow the example and adapt it to your customized properties. Open3D supports conversion from and to PyTorch tensors without memory any copy, see :ref:`/tutorial/core/tensor.ipynb#PyTorch-I/O-with-DLPack-memory-map`. This can be use to leverage PyTorch's capabilities such as automatic differentiation and other operators. diff --git a/docs/tutorial/t_reconstruction_system/dense_slam.rst b/docs/tutorial/t_reconstruction_system/dense_slam.rst index b11a76ed374..84b2c382fca 100644 --- a/docs/tutorial/t_reconstruction_system/dense_slam.rst +++ b/docs/tutorial/t_reconstruction_system/dense_slam.rst @@ -13,8 +13,8 @@ In a SLAM system, we maintain a ``model`` built upon a :ref:`voxel_block_grid`, .. literalinclude:: ../../../examples/python/t_reconstruction_system/dense_slam.py :language: python - :lineno-start: 45 - :lines: 27,46-54 + :lineno-start: 26 + :lines: 8,27-35 Frame-to-model tracking ```````````````````````` @@ -22,8 +22,8 @@ The frame-to-model tracking runs in a loop: .. literalinclude:: ../../../examples/python/t_reconstruction_system/dense_slam.py :language: python - :lineno-start: 57 - :lines: 27,58-78 + :lineno-start: 38 + :lines: 8,39,42-61 where we iteratively update the synthesized frame via ray-casting from the model, and perform the tensor version of :ref:`/tutorial/pipelines/rgbd_odometry.ipynb` between the input frame and the synthesized frame. @@ -44,7 +44,7 @@ If all above have been correctly set but still no luck, please file an issue. **Q**: So WHY did my tracking fail? -**A**: For the front end, we are using direct RGB-D odometry. Comparing to feature-based odometry, RGB-D odometry is more accurate when it completes successfully but is less robust. We will add support for feature-based tracking in the future. For the backend, unlike our offline reconstruction system, we do not detect loop closures, and do not perform pose graph optimization or bundle adjustment at the moment. +**A**: For the front end, we are using direct RGB-D odometry. Compared to feature-based odometry, RGB-D odometry is more accurate when it completes successfully but is less robust. We will add support for feature-based tracking in the future. For the backend, unlike our offline reconstruction system, we do not detect loop closures, and do not perform pose graph optimization or bundle adjustment at the moment. **Q**: Why don't you implement loop closure or relocalization? diff --git a/docs/tutorial/t_reconstruction_system/index.rst b/docs/tutorial/t_reconstruction_system/index.rst index e00d323e410..57fb76950b1 100644 --- a/docs/tutorial/t_reconstruction_system/index.rst +++ b/docs/tutorial/t_reconstruction_system/index.rst @@ -25,7 +25,7 @@ Getting the example code .. code-block:: sh - # Activate your conda enviroment, where you have installed open3d pip package. + # Activate your conda environment, where you have installed open3d pip package. # Clone the Open3D github repository and go to the example. cd examples/python/t_reconstruction_system/ @@ -40,7 +40,7 @@ Running the example with default dataset. # which is ``lounge`` dataset from stanford. python dense_slam_gui.py -It is recommended to use CUDA if avaialble. +It is recommended to use CUDA if available. .. code-block:: sh @@ -49,7 +49,7 @@ It is recommended to use CUDA if avaialble. python dense_slam_gui.py --device 'cuda:0' Changing the default dataset. -One may change the default dataset to other avaialble datasets. +One may change the default dataset to other available datasets. Currently the following datasets are available: 1. Lounge (keyword: ``lounge``) (Default) diff --git a/docs/tutorial/t_reconstruction_system/integration.rst b/docs/tutorial/t_reconstruction_system/integration.rst index 900e9bcad43..417f723205d 100644 --- a/docs/tutorial/t_reconstruction_system/integration.rst +++ b/docs/tutorial/t_reconstruction_system/integration.rst @@ -12,8 +12,8 @@ In the activation step, we first locate blocks that contain points unprojected f .. literalinclude:: ../../../examples/python/t_reconstruction_system/integrate.py :language: python - :lineno-start: 52 - :lines: 52-54 + :lineno-start: 51 + :lines: 8,52-54 Integration ```````````` @@ -23,24 +23,24 @@ We may use optimized functions, along with raw depth images with calibration par .. literalinclude:: ../../../examples/python/t_reconstruction_system/integrate.py :language: python - :lineno-start: 58 - :lines: 58-60 + :lineno-start: 55 + :lines: 8,56-63 Currently, to use our optimized function, we assume the below combinations of data types, in the order of ``depth image``, ``color image``, ``tsdf in voxel grid``, ``weight in voxel grid``, ``color in voxel grid`` in CPU .. literalinclude:: ../../../cpp/open3d/t/geometry/kernel/VoxelBlockGridCPU.cpp :language: cpp - :lineno-start: 229 - :lines: 230-236 + :lineno-start: 212 + :lines: 212-218 and CUDA .. literalinclude:: ../../../cpp/open3d/t/geometry/kernel/VoxelBlockGridCUDA.cu :language: cpp - :lineno-start: 255 - :lines: 256-262 + :lineno-start: 238 + :lines: 238-244 -For more generalized functionalities, you may extend the macros and/or the kernel functions and compile Open3D from scratch achieve the maximal performance (~100Hz on a GTX 1070), or follow :ref:`customized_integration` and implement a fast prototype (~25Hz). +For more generalized functionalities, you may extend the macros and/or the kernel functions and compile Open3D from scratch to achieve the maximal performance (~100Hz on a GTX 1070), or follow :ref:`customized_integration` and implement a fast prototype (~25Hz). Surface extraction `````````````````` @@ -48,10 +48,10 @@ You may use the provided APIs to extract surface points. .. literalinclude:: ../../../examples/python/t_reconstruction_system/integrate.py :language: python - :lineno-start: 106 - :lines: 106-110 + :lineno-start: 105 + :lines: 8, 106-110 -Note ``extract_triangle_mesh`` applies marching cubes and generate mesh. ``extract_point_cloud`` uses the similar algorithm, but skips the triangle face generation step. +Note ``extract_triangle_mesh`` applies marching cubes and generates mesh. ``extract_point_cloud`` uses a similar algorithm, but skips the triangle face generation step. Save and load `````````````` @@ -60,7 +60,7 @@ The voxel block grids can be saved to and loaded from `.npz` files that are acce .. literalinclude:: ../../../examples/python/t_reconstruction_system/integrate.py :language: python :lineno-start: 47 - :lines: 27,48,98 + :lines: 8,48,98 The ``.npz`` file of the aforementioned voxel block grid contains the following entries: diff --git a/docs/tutorial/t_reconstruction_system/ray_casting.rst b/docs/tutorial/t_reconstruction_system/ray_casting.rst index ca74ee98b02..3809a7941a7 100644 --- a/docs/tutorial/t_reconstruction_system/ray_casting.rst +++ b/docs/tutorial/t_reconstruction_system/ray_casting.rst @@ -12,19 +12,19 @@ We provide optimized conventional rendering, and basic support for customized re Conventional rendering `````````````````````` -From a reconstructed voxel block grid from :ref:`optimized_integration`, we can efficiently render the scene given the input depth as a rough range estimate. +From a reconstructed voxel block grid :code:`vbg` from :ref:`optimized_integration`, we can efficiently render the scene given the input depth as a rough range estimate. .. literalinclude:: ../../../examples/python/t_reconstruction_system/ray_casting.py :language: python - :lineno-start: 76 - :lines: 27,77-92 + :lineno-start: 68 + :lines: 8,69-82 The results could be directly obtained and visualized by .. literalinclude:: ../../../examples/python/t_reconstruction_system/ray_casting.py :language: python - :lineno-start: 90 - :lines: 27,91,93-95,105-112 + :lineno-start: 83 + :lines: 8,84,86-88,98-105 Customized rendering ````````````````````` @@ -32,7 +32,7 @@ In customized rendering, we manually perform trilinear-interpolation by accessin .. literalinclude:: ../../../examples/python/t_reconstruction_system/ray_casting.py :language: python - :lineno-start: 97 - :lines: 27,98-103,114-115 + :lineno-start: 90 + :lines: 8,91-96,107-108 Since the output is rendered via indices, the rendering process could be rewritten in differentiable engines like PyTorch seamlessly via :ref:`/tutorial/core/tensor.ipynb#PyTorch-I/O-with-DLPack-memory-map`. diff --git a/docs/tutorial/t_reconstruction_system/voxel_block_grid.rst b/docs/tutorial/t_reconstruction_system/voxel_block_grid.rst index 0836bd471f0..cf8eed1f2fa 100644 --- a/docs/tutorial/t_reconstruction_system/voxel_block_grid.rst +++ b/docs/tutorial/t_reconstruction_system/voxel_block_grid.rst @@ -14,21 +14,21 @@ A voxel block grid can be constructed by: .. literalinclude:: ../../../examples/python/t_reconstruction_system/integrate.py :language: python - :lineno-start: 56 - :lines: 27,57-74 + :lineno-start: 27 + :lines: 8,28-45 -In this example, the multi-value hash map has key shape shape ``(3,)`` and dtype ``int32``. The hash map values are organized as a structure of array (SoA). The hash map values include: +In this example, the multi-value hash map has key of shape ``(3,)`` and dtype ``float32``. The hash map values are organized as a structure of array (SoA). The hash map values include: By default it contains: -- Truncated Signed Distance Function (TSDF) of element shape ``(8, 8, 8, 1)`` -- Weight of element shape ``(8, 8, 8, 1)`` -- (Optionally) RGB color of element shape ``(8, 8, 8, 3)`` +- Truncated Signed Distance Function (TSDF) of element shape ``(16, 16, 16, 1)`` +- Weight of element shape ``(16, 16, 16, 1)`` +- (Optionally) RGB color of element shape ``(16, 16, 16, 3)`` -where ``8`` stands for the local voxel block grid resolution. +where ``16`` stands for the local voxel block grid resolution. By convention, we use ``3.0 / 512`` as the voxel resolution. This spatial resolution is equivalent to representing a ``3m x 3m x 3m`` (m = meter) room with a dense ``512 x 512 x 512`` voxel grid. -The voxel block grid is optimized to run fast on GPU. On CPU the it runs slower. Empirically, we reserve ``100000`` such blocks for a living room-scale scene to avoid frequent rehashing. +The voxel block grid is optimized to run fast on GPU. On CPU, it runs slower. Empirically, we reserve ``50000`` such blocks for a living room-scale scene to avoid frequent rehashing. -You can always customize your own properties, e.g., ``intensity`` of element shape ``(8, 8, 8, 1)`` in ``float32``, ``label`` of element shape ``(8, 8, 8, 1)`` in ``int32``, etc. To know how to process the data, please refer to :ref:`customized_integration`. +You can always customize your own properties, e.g., ``intensity`` of element shape ``(16, 16, 16, 1)`` in ``float32``, ``label`` of element shape ``(16, 16, 16, 1)`` in ``int32``, etc. To know how to process the data, please refer to :ref:`customized_integration`. diff --git a/docs/tutorial/visualization/cpu_rendering.rst b/docs/tutorial/visualization/cpu_rendering.rst index 6fb7ec938a3..f504382469e 100644 --- a/docs/tutorial/visualization/cpu_rendering.rst +++ b/docs/tutorial/visualization/cpu_rendering.rst @@ -46,7 +46,7 @@ Here are the different ways to do that: # In Python code import os - os.environ['EGL_PLATFORM'] = 'surfaceless' # Ubunu 20.04+ + os.environ['EGL_PLATFORM'] = 'surfaceless' # Ubuntu 20.04+ os.environ['OPEN3D_CPU_RENDERING'] = 'true' # Ubuntu 18.04 import open3d as o3d diff --git a/docs/tutorial/visualization/interactive_visualization.rst b/docs/tutorial/visualization/interactive_visualization.rst index d886466e568..8dfb03edebb 100644 --- a/docs/tutorial/visualization/interactive_visualization.rst +++ b/docs/tutorial/visualization/interactive_visualization.rst @@ -3,13 +3,16 @@ Interactive visualization ------------------------------------- -This tutorial introduces user interaction features of the visualizer window. +This tutorial introduces user interaction features of the visualizer window provided by:- + +#. ``open3d.visualization.draw_geometries_with_editing`` +#. ``open3d.visualization.VisualizerWithEditing`` .. literalinclude:: ../../../examples/python/visualization/interactive_visualization.py :language: python - :lineno-start: 27 - :lines: 27- + :start-at: # examples/python/visualization/interactive_visualization.py :linenos: + :lineno-match: This script executes two applications of user interaction: ``demo_crop_geometry`` and ``demo_manual_registration``. @@ -20,9 +23,9 @@ Crop geometry .. literalinclude:: ../../../examples/python/visualization/interactive_visualization.py :language: python - :lineno-start: 37 - :lines: 37-51 + :pyobject: demo_crop_geometry :linenos: + :lineno-match: This function simply reads a point cloud and calls ``draw_geometries_with_editing``. This function provides vertex selection and cropping. @@ -58,27 +61,30 @@ To finish selection mode, press ``F`` to switch to freeview mode. Manual registration ````````````````````````````````````````````` -Select correspondences -===================================== - The following script registers two point clouds using point-to-point ICP. It gets initial alignment via user interaction. +Prepare data +===================================== + .. literalinclude:: ../../../examples/python/visualization/interactive_visualization.py :language: python - :lineno-start: 61 - :lines: 61-76 + :pyobject: prepare_data :linenos: + :lineno-match: -The script reads two point clouds, and visualizes the point clouds before alignment. +This function reads two point clouds, and visualizes the point clouds before performing manual alignment. .. image:: ../../_static/visualization/interactive_visualization/manual_icp_initial.png :width: 400px +Select correspondences +===================================== + .. literalinclude:: ../../../examples/python/visualization/interactive_visualization.py :language: python - :lineno-start: 52 - :lines: 52-60 + :pyobject: pick_points :linenos: + :lineno-match: The function ``pick_points(pcd)`` makes an instance of ``VisualizerWithEditing``. To mimic ``draw_geometries``, it creates windows, adds the geometry, visualizes the geometry, and then terminates. A novel interface function from ``VisualizerWithEditing`` is ``get_picked_points()`` that returns the indices of user-picked vertices. @@ -115,9 +121,9 @@ Registration using user correspondences .. literalinclude:: ../../../examples/python/visualization/interactive_visualization.py :language: python - :lineno-start: 77 - :lines: 77-110 + :pyobject: register_via_correspondences :linenos: + :lineno-match: The later part of the demo computes an initial transformation based on the user-provided correspondences. This script builds pairs of correspondences using ``Vector2iVector(corr)``. It utilizes ``TransformationEstimationPointToPoint.compute_transformation`` to compute the initial transformation from the correspondences. The initial transformation is refined using ``registration_icp``. diff --git a/docs/tutorial/visualization/non_blocking_visualization.rst b/docs/tutorial/visualization/non_blocking_visualization.rst index c76a020abd8..9d3e932646a 100644 --- a/docs/tutorial/visualization/non_blocking_visualization.rst +++ b/docs/tutorial/visualization/non_blocking_visualization.rst @@ -28,6 +28,7 @@ This rendering loop can be readily customized. For example, a custom loop can be vis = Visualizer() vis.create_window() + vis.add_geometry(geometry) for i in range(icp_iteration): # do ICP single iteration # transform geometry using ICP @@ -39,7 +40,9 @@ The full script implementing this idea is displayed below. .. literalinclude:: ../../../examples/python/visualization/non_blocking_visualization.py :language: python + :start-at: # examples/python/visualization/non_blocking_visualization.py :linenos: + :lineno-match: The following sections explain this script. @@ -48,9 +51,9 @@ Prepare example data .. literalinclude:: ../../../examples/python/visualization/non_blocking_visualization.py :language: python - :lineno-start: 14 - :lines: 14-27 + :pyobject: prepare_data :linenos: + :lineno-match: This part reads two point clouds and downsamples them. The source point cloud is intentionally transformed for the misalignment. Both point clouds are flipped for better visualization. @@ -59,9 +62,10 @@ Initialize Visualizer class .. literalinclude:: ../../../examples/python/visualization/non_blocking_visualization.py :language: python - :lineno-start: 29 - :lines: 29-35 + :start-at: def demo_non_blocking_visualization(): + :end-at: save_image = False :linenos: + :lineno-match: These lines make an instance of the visualizer class, open a visualizer window, and add two geometries to the visualizer. @@ -70,9 +74,10 @@ Transform geometry and visualize it .. literalinclude:: ../../../examples/python/visualization/non_blocking_visualization.py :language: python - :lineno-start: 37 - :lines: 37-49 + :start-at: for i in range(icp_iteration): + :end-at: vis.destroy_window() :linenos: + :lineno-match: This script calls ``registration_icp`` for every iteration. Note that it explicitly forces only one ICP iteration via ``ICPConvergenceCriteria(max_iteration = 1)``. This is a trick to retrieve a slight pose update from a single ICP iteration. After ICP, source geometry is transformed accordingly. diff --git a/examples/python/reconstruction_system/debug/visualize_fragments.py b/examples/python/reconstruction_system/debug/visualize_fragments.py index c969a86e074..2c05bd19142 100644 --- a/examples/python/reconstruction_system/debug/visualize_fragments.py +++ b/examples/python/reconstruction_system/debug/visualize_fragments.py @@ -5,7 +5,7 @@ # SPDX-License-Identifier: MIT # ---------------------------------------------------------------------------- -# examples/python/reconstruction_system/debug/visualize_fragment.py +# examples/python/reconstruction_system/debug/visualize_fragments.py import argparse import json diff --git a/examples/python/reconstruction_system/make_fragments.py b/examples/python/reconstruction_system/make_fragments.py index 138fe567aaf..dc4e59d0780 100644 --- a/examples/python/reconstruction_system/make_fragments.py +++ b/examples/python/reconstruction_system/make_fragments.py @@ -8,6 +8,7 @@ # examples/python/reconstruction_system/make_fragments.py import math +import multiprocessing import os, sys import numpy as np import open3d as o3d @@ -172,13 +173,14 @@ def run(config): math.ceil(float(n_files) / config['n_frames_per_fragment'])) if config["python_multi_threading"] is True: - from joblib import Parallel, delayed - import multiprocessing - import subprocess - MAX_THREAD = min(multiprocessing.cpu_count(), n_fragments) - Parallel(n_jobs=MAX_THREAD)(delayed(process_single_fragment)( - fragment_id, color_files, depth_files, n_files, n_fragments, config) - for fragment_id in range(n_fragments)) + max_workers = min(max(1, multiprocessing.cpu_count() - 1), n_fragments) + # Prevent over allocation of open mp threads in child processes + os.environ['OMP_NUM_THREADS'] = '1' + mp_context = multiprocessing.get_context('spawn') + with mp_context.Pool(processes=max_workers) as pool: + args = [(fragment_id, color_files, depth_files, n_files, + n_fragments, config) for fragment_id in range(n_fragments)] + pool.starmap(process_single_fragment, args) else: for fragment_id in range(n_fragments): process_single_fragment(fragment_id, color_files, depth_files, diff --git a/examples/python/reconstruction_system/refine_registration.py b/examples/python/reconstruction_system/refine_registration.py index 85cb5311624..bd555b150f9 100644 --- a/examples/python/reconstruction_system/refine_registration.py +++ b/examples/python/reconstruction_system/refine_registration.py @@ -7,9 +7,12 @@ # examples/python/reconstruction_system/refine_registration.py +import multiprocessing +import os +import sys + import numpy as np import open3d as o3d -import os, sys pyexample_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(pyexample_path) @@ -163,29 +166,28 @@ def make_posegraph_for_refined_scene(ply_file_names, config): s = edge.source_node_id t = edge.target_node_id matching_results[s * n_files + t] = \ - matching_result(s, t, edge.transformation) - - if config["python_multi_threading"] == True: - from joblib import Parallel, delayed - import multiprocessing - import subprocess - MAX_THREAD = min(multiprocessing.cpu_count(), - max(len(pose_graph.edges), 1)) - results = Parallel(n_jobs=MAX_THREAD)( - delayed(register_point_cloud_pair)( - ply_file_names, matching_results[r].s, matching_results[r].t, - matching_results[r].transformation, config) - for r in matching_results) + matching_result(s, t, edge.transformation) + + if config["python_multi_threading"] is True: + os.environ['OMP_NUM_THREADS'] = '1' + max_workers = max( + 1, min(multiprocessing.cpu_count() - 1, len(pose_graph.edges))) + mp_context = multiprocessing.get_context('spawn') + with mp_context.Pool(processes=max_workers) as pool: + args = [(ply_file_names, v.s, v.t, v.transformation, config) + for k, v in matching_results.items()] + results = pool.starmap(register_point_cloud_pair, args) + for i, r in enumerate(matching_results): matching_results[r].transformation = results[i][0] matching_results[r].information = results[i][1] else: for r in matching_results: (matching_results[r].transformation, - matching_results[r].information) = \ - register_point_cloud_pair(ply_file_names, - matching_results[r].s, matching_results[r].t, - matching_results[r].transformation, config) + matching_results[r].information) = \ + register_point_cloud_pair(ply_file_names, + matching_results[r].s, matching_results[r].t, + matching_results[r].transformation, config) pose_graph_new = o3d.pipelines.registration.PoseGraph() odometry = np.identity(4) diff --git a/examples/python/reconstruction_system/register_fragments.py b/examples/python/reconstruction_system/register_fragments.py index 82e31fba9d7..f7d3c72e664 100644 --- a/examples/python/reconstruction_system/register_fragments.py +++ b/examples/python/reconstruction_system/register_fragments.py @@ -7,9 +7,12 @@ # examples/python/reconstruction_system/register_fragments.py +import multiprocessing +import os +import sys + import numpy as np import open3d as o3d -import os, sys pyexample_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(pyexample_path) @@ -158,16 +161,16 @@ def make_posegraph_for_scene(ply_file_names, config): for t in range(s + 1, n_files): matching_results[s * n_files + t] = matching_result(s, t) - if config["python_multi_threading"] == True: - from joblib import Parallel, delayed - import multiprocessing - import subprocess - MAX_THREAD = min(multiprocessing.cpu_count(), - max(len(matching_results), 1)) - results = Parallel(n_jobs=MAX_THREAD)(delayed( - register_point_cloud_pair)(ply_file_names, matching_results[r].s, - matching_results[r].t, config) - for r in matching_results) + if config["python_multi_threading"] is True: + os.environ['OMP_NUM_THREADS'] = '1' + max_workers = max( + 1, min(multiprocessing.cpu_count() - 1, len(matching_results))) + mp_context = multiprocessing.get_context('spawn') + with mp_context.Pool(processes=max_workers) as pool: + args = [(ply_file_names, v.s, v.t, config) + for k, v in matching_results.items()] + results = pool.starmap(register_point_cloud_pair, args) + for i, r in enumerate(matching_results): matching_results[r].success = results[i][0] matching_results[r].transformation = results[i][1] @@ -175,9 +178,9 @@ def make_posegraph_for_scene(ply_file_names, config): else: for r in matching_results: (matching_results[r].success, matching_results[r].transformation, - matching_results[r].information) = \ - register_point_cloud_pair(ply_file_names, - matching_results[r].s, matching_results[r].t, config) + matching_results[r].information) = \ + register_point_cloud_pair(ply_file_names, + matching_results[r].s, matching_results[r].t, config) for r in matching_results: if matching_results[r].success: diff --git a/examples/python/reconstruction_system/sensors/realsense_helper.py b/examples/python/reconstruction_system/sensors/realsense_helper.py index 65c68014553..5e37b0c50bb 100644 --- a/examples/python/reconstruction_system/sensors/realsense_helper.py +++ b/examples/python/reconstruction_system/sensors/realsense_helper.py @@ -5,7 +5,7 @@ # SPDX-License-Identifier: MIT # ---------------------------------------------------------------------------- -# examples/python/reconstruction_system/sensors/realsense_pcd_visualizer.py +# examples/python/reconstruction_system/sensors/realsense_helper.py # pyrealsense2 is required. # Please see instructions in https://github.com/IntelRealSense/librealsense/tree/master/wrappers/python diff --git a/examples/python/t_reconstruction_system/common.py b/examples/python/t_reconstruction_system/common.py index 1831545388b..d6331920d87 100644 --- a/examples/python/t_reconstruction_system/common.py +++ b/examples/python/t_reconstruction_system/common.py @@ -5,7 +5,7 @@ # SPDX-License-Identifier: MIT # ---------------------------------------------------------------------------- -# examples/python/reconstruction_system/common.py +# examples/python/t_reconstruction_system/common.py import open3d as o3d @@ -95,7 +95,7 @@ def get_default_dataset(config): def load_depth_file_names(config): if not os.path.exists(config.path_dataset): print( - 'Path \'{}\' not found.'.format(config.path_dataset), + f"Path '{config.path_dataset}' not found.", 'Please provide --path_dataset in the command line or the config file.' ) return [], [] @@ -106,7 +106,7 @@ def load_depth_file_names(config): depth_file_names = glob.glob(os.path.join(depth_folder, '*.png')) n_depth = len(depth_file_names) if n_depth == 0: - print('Depth image not found in {}, abort!'.format(depth_folder)) + print(f'Depth image not found in {depth_folder}, abort!') return [] return sorted(depth_file_names) diff --git a/examples/python/t_reconstruction_system/config.py b/examples/python/t_reconstruction_system/config.py index 2ca1305e2fc..59b9ec40e2f 100644 --- a/examples/python/t_reconstruction_system/config.py +++ b/examples/python/t_reconstruction_system/config.py @@ -7,8 +7,8 @@ import os -import argparse import configargparse +import open3d as o3d class ConfigParser(configargparse.ArgParser): @@ -109,8 +109,8 @@ def __init__(self): integration_parser = self.add_argument_group('integration') integration_parser.add( - '--integrate_color', action=argparse.BooleanOptionalAction, - default=True, help='Volumetric integration mode.') + '--integrate_color', action='store_true', + default=False, help='Volumetric integration mode.') integration_parser.add( '--voxel_size', type=float, help='Voxel size in meter for volumetric integration.') @@ -142,7 +142,7 @@ def get_config(self): 'Fallback to hybrid odometry.') config.odometry_method = 'hybrid' - if config.engine == 'tensor': + elif config.engine == 'tensor': if config.icp_method == 'generalized': print('Tensor engine does not support generalized ICP.', 'Fallback to colored ICP.') @@ -153,6 +153,13 @@ def get_config(self): 'Disabled.') config.multiprocessing = False + if (config.device.lower().startswith('cuda') and + (not o3d.core.cuda.is_available())): + print( + 'Open3d not built with cuda support or no cuda device available. ', + 'Fallback to CPU.') + config.device = 'CPU:0' + return config diff --git a/examples/python/t_reconstruction_system/dense_slam.py b/examples/python/t_reconstruction_system/dense_slam.py index e22d6eacfa8..374b6dce91c 100644 --- a/examples/python/t_reconstruction_system/dense_slam.py +++ b/examples/python/t_reconstruction_system/dense_slam.py @@ -5,7 +5,7 @@ # SPDX-License-Identifier: MIT # ---------------------------------------------------------------------------- -# examples/python/t_reconstruction_system/ray_casting.py +# examples/python/t_reconstruction_system/dense_slam.py # P.S. This example is used in documentation, so, please ensure the changes are # synchronized. @@ -91,7 +91,7 @@ def slam(depth_file_names, color_file_names, intrinsic, config): # Extract RGB-D frames and intrinsic from bag file. if config.path_dataset.endswith(".bag"): assert os.path.isfile( - config.path_dataset), (f"File {config.path_dataset} not found.") + config.path_dataset), f"File {config.path_dataset} not found." print("Extracting frames from RGBD video file") config.path_dataset, config.path_intrinsic, config.depth_scale = extract_rgbd_frames( config.path_dataset) diff --git a/examples/python/t_reconstruction_system/dense_slam_gui.py b/examples/python/t_reconstruction_system/dense_slam_gui.py index a4e68526754..24744b9e87d 100644 --- a/examples/python/t_reconstruction_system/dense_slam_gui.py +++ b/examples/python/t_reconstruction_system/dense_slam_gui.py @@ -17,7 +17,7 @@ from config import ConfigParser -import os, sys +import os import numpy as np import threading import time @@ -459,7 +459,7 @@ def update_main(self): # Extract RGB-D frames and intrinsic from bag file. if config.path_dataset.endswith(".bag"): assert os.path.isfile( - config.path_dataset), (f"File {config.path_dataset} not found.") + config.path_dataset), f"File {config.path_dataset} not found." print("Extracting frames from RGBD video file") config.path_dataset, config.path_intrinsic, config.depth_scale = extract_rgbd_frames( config.path_dataset) diff --git a/examples/python/t_reconstruction_system/integrate_custom.py b/examples/python/t_reconstruction_system/integrate_custom.py index c6b8df3c7f4..8b352780b04 100644 --- a/examples/python/t_reconstruction_system/integrate_custom.py +++ b/examples/python/t_reconstruction_system/integrate_custom.py @@ -5,139 +5,137 @@ # SPDX-License-Identifier: MIT # ---------------------------------------------------------------------------- -# examples/python/t_reconstruction_system/ray_casting.py +# examples/python/t_reconstruction_system/integrate_custom.py # P.S. This example is used in documentation, so, please ensure the changes are # synchronized. import os -import numpy as np +import time + import open3d as o3d import open3d.core as o3c -import time -import matplotlib.pyplot as plt +from tqdm import tqdm +from common import get_default_dataset, load_rgbd_file_names, load_depth_file_names, load_intrinsic, load_extrinsics, extract_rgbd_frames from config import ConfigParser -from common import get_default_dataset, load_rgbd_file_names, load_depth_file_names, save_poses, load_intrinsic, load_extrinsics, extract_rgbd_frames - -from tqdm import tqdm def integrate(depth_file_names, color_file_names, intrinsic, extrinsics, config): if os.path.exists(config.path_npz): - print('Voxel block grid npz file {} found, trying to load...'.format( - config.path_npz)) + print( + f'Voxel block grid npz file {config.path_npz} found, trying to load...' + ) vbg = o3d.t.geometry.VoxelBlockGrid.load(config.path_npz) print('Loading finished.') + return vbg + + print( + f'Voxel block grid npz file {config.path_npz} not found, trying to integrate...' + ) + + n_files = len(depth_file_names) + device = o3d.core.Device(config.device) + + voxel_size = 3.0 / 512 + trunc = voxel_size * 4 + res = 8 + + if config.integrate_color: + vbg = o3d.t.geometry.VoxelBlockGrid( + ('tsdf', 'weight', 'color'), + (o3c.float32, o3c.float32, o3c.float32), ((1), (1), (3)), 3.0 / 512, + 8, 100000, device) else: - print('Voxel block grid npz file {} not found, trying to integrate...'. - format(config.path_npz)) + vbg = o3d.t.geometry.VoxelBlockGrid( + ('tsdf', 'weight'), (o3c.float32, o3c.float32), ((1), (1)), + 3.0 / 512, 8, 100000, device) - n_files = len(depth_file_names) - device = o3d.core.Device(config.device) + start = time.time() + for i in tqdm(range(n_files)): + depth = o3d.t.io.read_image(depth_file_names[i]).to(device) + extrinsic = extrinsics[i] - voxel_size = 3.0 / 512 - trunc = voxel_size * 4 - res = 8 + start = time.time() + # Get active frustum block coordinates from input + frustum_block_coords = vbg.compute_unique_block_coordinates( + depth, intrinsic, extrinsic, config.depth_scale, config.depth_max) + # Activate them in the underlying hash map (may have been inserted) + vbg.hashmap().activate(frustum_block_coords) - if config.integrate_color: - vbg = o3d.t.geometry.VoxelBlockGrid( - ('tsdf', 'weight', 'color'), - (o3c.float32, o3c.float32, o3c.float32), ((1), (1), (3)), - 3.0 / 512, 8, 100000, device) - else: - vbg = o3d.t.geometry.VoxelBlockGrid( - ('tsdf', 'weight'), (o3c.float32, o3c.float32), ((1), (1)), - 3.0 / 512, 8, 100000, device) + # Find buf indices in the underlying engine + buf_indices, masks = vbg.hashmap().find(frustum_block_coords) + o3d.core.cuda.synchronize() + end = time.time() + + start = time.time() + voxel_coords, voxel_indices = vbg.voxel_coordinates_and_flattened_indices( + buf_indices) + o3d.core.cuda.synchronize() + end = time.time() + # Now project them to the depth and find association + # (3, N) -> (2, N) start = time.time() - for i in tqdm(range(n_files)): - depth = o3d.t.io.read_image(depth_file_names[i]).to(device) - extrinsic = extrinsics[i] - - start = time.time() - # Get active frustum block coordinates from input - frustum_block_coords = vbg.compute_unique_block_coordinates( - depth, intrinsic, extrinsic, config.depth_scale, - config.depth_max) - # Activate them in the underlying hash map (may have been inserted) - vbg.hashmap().activate(frustum_block_coords) - - # Find buf indices in the underlying engine - buf_indices, masks = vbg.hashmap().find(frustum_block_coords) - o3d.core.cuda.synchronize() - end = time.time() - - start = time.time() - voxel_coords, voxel_indices = vbg.voxel_coordinates_and_flattened_indices( - buf_indices) - o3d.core.cuda.synchronize() - end = time.time() - - # Now project them to the depth and find association - # (3, N) -> (2, N) - start = time.time() - extrinsic_dev = extrinsic.to(device, o3c.float32) - xyz = extrinsic_dev[:3, :3] @ voxel_coords.T() + extrinsic_dev[:3, - 3:] - - intrinsic_dev = intrinsic.to(device, o3c.float32) - uvd = intrinsic_dev @ xyz - d = uvd[2] - u = (uvd[0] / d).round().to(o3c.int64) - v = (uvd[1] / d).round().to(o3c.int64) - o3d.core.cuda.synchronize() - end = time.time() - - start = time.time() - mask_proj = (d > 0) & (u >= 0) & (v >= 0) & (u < depth.columns) & ( - v < depth.rows) - - v_proj = v[mask_proj] - u_proj = u[mask_proj] - d_proj = d[mask_proj] - depth_readings = depth.as_tensor()[v_proj, u_proj, 0].to( - o3c.float32) / config.depth_scale - sdf = depth_readings - d_proj - - mask_inlier = (depth_readings > 0) \ - & (depth_readings < config.depth_max) \ - & (sdf >= -trunc) - - sdf[sdf >= trunc] = trunc - sdf = sdf / trunc - o3d.core.cuda.synchronize() - end = time.time() - - start = time.time() - weight = vbg.attribute('weight').reshape((-1, 1)) - tsdf = vbg.attribute('tsdf').reshape((-1, 1)) - - valid_voxel_indices = voxel_indices[mask_proj][mask_inlier] - w = weight[valid_voxel_indices] - wp = w + 1 - - tsdf[valid_voxel_indices] \ - = (tsdf[valid_voxel_indices] * w + - sdf[mask_inlier].reshape(w.shape)) / (wp) - if config.integrate_color: - color = o3d.t.io.read_image(color_file_names[i]).to(device) - color_readings = color.as_tensor()[v_proj, - u_proj].to(o3c.float32) - - color = vbg.attribute('color').reshape((-1, 3)) - color[valid_voxel_indices] \ - = (color[valid_voxel_indices] * w + - color_readings[mask_inlier]) / (wp) - - weight[valid_voxel_indices] = wp - o3d.core.cuda.synchronize() - end = time.time() - - print('Saving to {}...'.format(config.path_npz)) - vbg.save(config.path_npz) - print('Saving finished') + extrinsic_dev = extrinsic.to(device, o3c.float32) + xyz = extrinsic_dev[:3, :3] @ voxel_coords.T() + extrinsic_dev[:3, 3:] + + intrinsic_dev = intrinsic.to(device, o3c.float32) + uvd = intrinsic_dev @ xyz + d = uvd[2] + u = (uvd[0] / d).round().to(o3c.int64) + v = (uvd[1] / d).round().to(o3c.int64) + o3d.core.cuda.synchronize() + end = time.time() + + start = time.time() + mask_proj = (d > 0) & (u >= 0) & (v >= 0) & (u < depth.columns) & ( + v < depth.rows) + + v_proj = v[mask_proj] + u_proj = u[mask_proj] + d_proj = d[mask_proj] + depth_readings = depth.as_tensor()[v_proj, u_proj, 0].to( + o3c.float32) / config.depth_scale + sdf = depth_readings - d_proj + + mask_inlier = (depth_readings > 0) \ + & (depth_readings < config.depth_max) \ + & (sdf >= -trunc) + + sdf[sdf >= trunc] = trunc + sdf = sdf / trunc + o3d.core.cuda.synchronize() + end = time.time() + + start = time.time() + weight = vbg.attribute('weight').reshape((-1, 1)) + tsdf = vbg.attribute('tsdf').reshape((-1, 1)) + + valid_voxel_indices = voxel_indices[mask_proj][mask_inlier] + w = weight[valid_voxel_indices] + wp = w + 1 + + tsdf[valid_voxel_indices] \ + = (tsdf[valid_voxel_indices] * w + + sdf[mask_inlier].reshape(w.shape)) / (wp) + if config.integrate_color: + color = o3d.t.io.read_image(color_file_names[i]).to(device) + color_readings = color.as_tensor()[v_proj, u_proj].to(o3c.float32) + + color = vbg.attribute('color').reshape((-1, 3)) + color[valid_voxel_indices] \ + = (color[valid_voxel_indices] * w + + color_readings[mask_inlier]) / (wp) + + weight[valid_voxel_indices] = wp + o3d.core.cuda.synchronize() + end = time.time() + + print(f'Saving to {config.path_npz}...') + vbg.save(config.path_npz) + print('Saving finished') return vbg @@ -170,7 +168,7 @@ def integrate(depth_file_names, color_file_names, intrinsic, extrinsics, # Extract RGB-D frames and intrinsic from bag file. if config.path_dataset.endswith(".bag"): assert os.path.isfile( - config.path_dataset), (f"File {config.path_dataset} not found.") + config.path_dataset), f"File {config.path_dataset} not found." print("Extracting frames from RGBD video file") config.path_dataset, config.path_intrinsic, config.depth_scale = extract_rgbd_frames( config.path_dataset) diff --git a/examples/python/t_reconstruction_system/pose_graph_optim.py b/examples/python/t_reconstruction_system/pose_graph_optim.py index 151f55a941f..94abaf34711 100644 --- a/examples/python/t_reconstruction_system/pose_graph_optim.py +++ b/examples/python/t_reconstruction_system/pose_graph_optim.py @@ -57,8 +57,8 @@ def _dicts2graph(self): for (i, j) in self.dict_edges: if not (i in self.dict_nodes) or not (j in self.dict_nodes): print( - 'Edge node ({} {}) not found, abort pose graph construction' - .format(i, j)) + f'Edge node ({i} {j}) not found, abort pose graph construction' + ) trans, info, is_loop = self.dict_edges[(i, j)] ki = nodes2indices[i] kj = nodes2indices[j] diff --git a/examples/python/t_reconstruction_system/ray_casting.py b/examples/python/t_reconstruction_system/ray_casting.py index 84af9a97af1..5bae7301a30 100644 --- a/examples/python/t_reconstruction_system/ray_casting.py +++ b/examples/python/t_reconstruction_system/ray_casting.py @@ -50,7 +50,7 @@ # Extract RGB-D frames and intrinsic from bag file. if config.path_dataset.endswith(".bag"): assert os.path.isfile( - config.path_dataset), (f"File {config.path_dataset} not found.") + config.path_dataset), f"File {config.path_dataset} not found." print("Extracting frames from RGBD video file") config.path_dataset, config.path_intrinsic, config.depth_scale = extract_rgbd_frames( config.path_dataset) diff --git a/examples/python/visualization/interactive_visualization.py b/examples/python/visualization/interactive_visualization.py index 29e98dd6c08..1b3ff5ed051 100644 --- a/examples/python/visualization/interactive_visualization.py +++ b/examples/python/visualization/interactive_visualization.py @@ -37,6 +37,15 @@ def draw_registration_result(source, target, transformation): o3d.visualization.draw_geometries([source_temp, target_temp]) +def prepare_data(): + pcd_data = o3d.data.DemoICPPointClouds() + source = o3d.io.read_point_cloud(pcd_data.paths[0]) + target = o3d.io.read_point_cloud(pcd_data.paths[2]) + print("Visualization of two point clouds before manual alignment") + draw_registration_result(source, target, np.identity(4)) + return source, target + + def pick_points(pcd): print("") print( @@ -53,29 +62,15 @@ def pick_points(pcd): return vis.get_picked_points() -def demo_manual_registration(): - print("Demo for manual ICP") - pcd_data = o3d.data.DemoICPPointClouds() - source = o3d.io.read_point_cloud(pcd_data.paths[0]) - target = o3d.io.read_point_cloud(pcd_data.paths[2]) - print("Visualization of two point clouds before manual alignment") - draw_registration_result(source, target, np.identity(4)) - - # pick points from two point clouds and builds correspondences - picked_id_source = pick_points(source) - picked_id_target = pick_points(target) - assert (len(picked_id_source) >= 3 and len(picked_id_target) >= 3) - assert (len(picked_id_source) == len(picked_id_target)) - corr = np.zeros((len(picked_id_source), 2)) - corr[:, 0] = picked_id_source - corr[:, 1] = picked_id_target - +def register_via_correspondences(source, target, source_points, target_points): + corr = np.zeros((len(source_points), 2)) + corr[:, 0] = source_points + corr[:, 1] = target_points # estimate rough transformation using correspondences print("Compute a rough transform using the correspondences given by user") p2p = o3d.pipelines.registration.TransformationEstimationPointToPoint() trans_init = p2p.compute_transformation(source, target, o3d.utility.Vector2iVector(corr)) - # point-to-point ICP for refinement print("Perform point-to-point ICP refinement") threshold = 0.03 # 3cm distance threshold @@ -83,6 +78,18 @@ def demo_manual_registration(): source, target, threshold, trans_init, o3d.pipelines.registration.TransformationEstimationPointToPoint()) draw_registration_result(source, target, reg_p2p.transformation) + + +def demo_manual_registration(): + print("Demo for manual ICP") + source, target = prepare_data() + + # pick points from two point clouds and builds correspondences + source_points = pick_points(source) + target_points = pick_points(target) + assert (len(source_points) >= 3 and len(target_points) >= 3) + assert (len(source_points) == len(target_points)) + register_via_correspondences(source, target, source_points, target_points) print("") diff --git a/examples/python/visualization/non_blocking_visualization.py b/examples/python/visualization/non_blocking_visualization.py index cc8bd74697c..04baaf85574 100644 --- a/examples/python/visualization/non_blocking_visualization.py +++ b/examples/python/visualization/non_blocking_visualization.py @@ -10,22 +10,27 @@ import open3d as o3d import numpy as np -if __name__ == "__main__": - o3d.utility.set_verbosity_level(o3d.utility.VerbosityLevel.Debug) + +def prepare_data(): pcd_data = o3d.data.DemoICPPointClouds() source_raw = o3d.io.read_point_cloud(pcd_data.paths[0]) target_raw = o3d.io.read_point_cloud(pcd_data.paths[1]) - source = source_raw.voxel_down_sample(voxel_size=0.02) target = target_raw.voxel_down_sample(voxel_size=0.02) + trans = [[0.862, 0.011, -0.507, 0.0], [-0.139, 0.967, -0.215, 0.7], [0.487, 0.255, 0.835, -1.4], [0.0, 0.0, 0.0, 1.0]] source.transform(trans) - flip_transform = [[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0], [0, 0, 0, 1]] source.transform(flip_transform) target.transform(flip_transform) + return source, target + +def demo_non_blocking_visualization(): + o3d.utility.set_verbosity_level(o3d.utility.VerbosityLevel.Debug) + + source, target = prepare_data() vis = o3d.visualization.Visualizer() vis.create_window() vis.add_geometry(source) @@ -46,4 +51,9 @@ if save_image: vis.capture_screen_image("temp_%04d.jpg" % i) vis.destroy_window() + o3d.utility.set_verbosity_level(o3d.utility.VerbosityLevel.Info) + + +if __name__ == '__main__': + demo_non_blocking_visualization()