Fix the textures loading in .glb file (#5443)

cpp/open3d/data/Dataset.cpp

@@ -582,6 +582,28 @@ FlightHelmetModel::FlightHelmetModel(const std::string& data_root)
                            "OcclusionRoughMetal.png"}};
 }
 
+AvocadoModel::AvocadoModel(const std::string& data_root)
+    : SingleDownloadDataset(
+              "AvocadoModel",
+              {"https://github.com/isl-org/open3d_downloads/releases/download/"
+               "20220301-data/AvocadoModel.glb"},
+              "829f96a0a3a7d5556e0a263ea0699217",
+              /*no_extract =*/true,
+              data_root) {
+    path_ = Dataset::GetExtractDir() + "/AvocadoModel.glb";
+}
+
+DamagedHelmetModel::DamagedHelmetModel(const std::string& data_root)
+    : SingleDownloadDataset(
+              "DamagedHelmetModel",
+              {"https://github.com/isl-org/open3d_downloads/releases/download/"
+               "20220301-data/DamagedHelmetModel.glb"},
+              "a3af6ad5a8329f22ba08b7f16e4a97d8",
+              /*no_extract =*/true,
+              data_root) {
+    path_ = Dataset::GetExtractDir() + "/DamagedHelmetModel.glb";
+}
+
 MetalTexture::MetalTexture(const std::string& data_root)
     : SingleDownloadDataset(
               "MetalTexture",

cpp/open3d/data/Dataset.h

@@ -643,6 +643,36 @@ class FlightHelmetModel : public SingleDownloadDataset {
     std::unordered_map<std::string, std::string> map_filename_to_path_;
 };
 
+/// \class AvocadoModel
+/// \brief Data class for `AvocadoModel` contains a avocado model file,
+/// along with material and PNG format embedded textures.
+class AvocadoModel : public SingleDownloadDataset {
+public:
+    AvocadoModel(const std::string& data_root = "");
+
+    /// \brief Returns path to the GLB format avocado model.
+    std::string GetPath() const { return path_; };
+
+private:
+    /// Path to the GLB format avocado model.
+    std::string path_;
+};
+
+/// \class DamagedHelmetModel
+/// \brief Data class for `DamagedHelmetModel` contains a damaged helmet model
+/// file, along with material and JPG format embedded textures.
+class DamagedHelmetModel : public SingleDownloadDataset {
+public:
+    DamagedHelmetModel(const std::string& data_root = "");
+
+    /// \brief Returns path to the GLB format damaged helmet model.
+    std::string GetPath() const { return path_; };
+
+private:
+    /// Path to the GLB format damaged helmet model.
+    std::string path_;
+};
+
 /// \class MetalTexture
 /// \brief Data class for `MetalTexture` contains albedo, normal, roughness and
 /// metallic texture files for metal based material.

cpp/open3d/io/ImageIO.cpp

@@ -102,5 +102,28 @@ bool WriteImage(const std::string &filename,
     return map_itr->second(filename, image, quality);
 }
 
+std::shared_ptr<geometry::Image> CreateImageFromMemory(
+        const std::string &image_format,
+        const unsigned char *image_data_ptr,
+        size_t image_data_size) {
+    auto image = std::make_shared<geometry::Image>();
+    ReadImageFromMemory(image_format, image_data_ptr, image_data_size, *image);
+    return image;
+}
+
+bool ReadImageFromMemory(const std::string &image_format,
+                         const unsigned char *image_data_ptr,
+                         size_t image_data_size,
+                         geometry::Image &image) {
+    if (image_format == "png") {
+        return ReadPNGFromMemory(image_data_ptr, image_data_size, image);
+    } else if (image_format == "jpg") {
+        return ReadJPGFromMemory(image_data_ptr, image_data_size, image);
+    } else {
+        utility::LogWarning("The format of {} is not supported", image_format);
+        return false;
+    }
+}
+
 }  // namespace io
 }  // namespace open3d

cpp/open3d/io/ImageIO.h

@@ -38,11 +38,28 @@ namespace io {
 std::shared_ptr<geometry::Image> CreateImageFromFile(
         const std::string &filename);
 
+/// Factory function to create an image from memory.
+std::shared_ptr<geometry::Image> CreateImageFromMemory(
+        const std::string &image_format,
+        const unsigned char *image_data_ptr,
+        size_t image_data_size);
+
 /// The general entrance for reading an Image from a file
 /// The function calls read functions based on the extension name of filename.
 /// \return return true if the read function is successful, false otherwise.
 bool ReadImage(const std::string &filename, geometry::Image &image);
 
+/// The general entrance for reading an Image from memory
+/// The function calls read functions based on format of image.
+/// \param image_format the format of image, "png" or "jpg".
+/// \param image_data_ptr the pointer to image data in memory.
+/// \param image_data_size the size of image data in memory.
+/// \return return true if the read function is successful, false otherwise.
+bool ReadImageFromMemory(const std::string &image_format,
+                         const unsigned char *image_data_ptr,
+                         size_t image_data_size,
+                         geometry::Image &image);
+
 constexpr int kOpen3DImageIODefaultQuality = -1;
 
 /// The general entrance for writing an Image to a file
@@ -71,5 +88,14 @@ bool WriteImageToJPG(const std::string &filename,
                      const geometry::Image &image,
                      int quality = kOpen3DImageIODefaultQuality);
 
+/// The general entrance for reading an Image from memory
+bool ReadPNGFromMemory(const unsigned char *image_data_ptr,
+                       size_t image_data_size,
+                       geometry::Image &image);
+
+bool ReadJPGFromMemory(const unsigned char *image_data_ptr,
+                       size_t image_data_size,
+                       geometry::Image &image);
+
 }  // namespace io
 }  // namespace open3d

cpp/open3d/io/file_format/FileASSIMP.cpp

@@ -77,34 +77,66 @@ struct TextureImages {
 };
 
 void LoadTextures(const std::string& filename,
-                  aiMaterial* mat,
+                  const aiScene* scene,
+                  const aiMaterial* mat,
                   TextureImages& maps) {
     // Retrieve textures
     std::string base_path =
             utility::filesystem::GetFileParentDirectory(filename);
 
-    auto texture_loader = [&base_path, &mat](
+    auto texture_loader = [&base_path, &scene, &mat](
                                   aiTextureType type,
                                   std::shared_ptr<geometry::Image>& img) {
         if (mat->GetTextureCount(type) > 0) {
             aiString path;
             mat->GetTexture(type, 0, &path);
-            std::string strpath(path.C_Str());
-            // normalize path separators
-            auto p_win = strpath.find("\\");
-            while (p_win != std::string::npos) {
-                strpath[p_win] = '/';
-                p_win = strpath.find("\\", p_win + 1);
-            }
-            // if absolute path convert to relative to base path
-            if (strpath.length() > 1 &&
-                (strpath[0] == '/' || strpath[1] == ':')) {
-                strpath = utility::filesystem::GetFileNameWithoutDirectory(
-                        strpath);
+            // If the texture is an embedded texture, use `GetEmbeddedTexture`.
+            if (auto texture = scene->GetEmbeddedTexture(path.C_Str())) {
+                if (texture->CheckFormat("png")) {
+                    auto image = io::CreateImageFromMemory(
+                            "png",
+                            reinterpret_cast<const unsigned char*>(
+                                    texture->pcData),
+                            texture->mWidth);
+                    if (image->HasData()) {
+                        img = image;
+                    }
+                } else if (texture->CheckFormat("jpg")) {
+                    auto image = io::CreateImageFromMemory(
+                            "jpg",
+                            reinterpret_cast<const unsigned char*>(
+                                    texture->pcData),
+                            texture->mWidth);
+                    if (image->HasData()) {
+                        img = image;
+                    }
+                }
+
+                else {
+                    utility::LogWarning(
+                            "This format of image is not supported.");
+                }
+
             }
-            auto image = io::CreateImageFromFile(base_path + strpath);
-            if (image->HasData()) {
-                img = image;
+            // Else, build the path to it.
+            else {
+                std::string strpath(path.C_Str());
+                // Normalize path separators.
+                auto p_win = strpath.find("\\");
+                while (p_win != std::string::npos) {
+                    strpath[p_win] = '/';
+                    p_win = strpath.find("\\", p_win + 1);
+                }
+                // If absolute path convert to relative to base path.
+                if (strpath.length() > 1 &&
+                    (strpath[0] == '/' || strpath[1] == ':')) {
+                    strpath = utility::filesystem::GetFileNameWithoutDirectory(
+                            strpath);
+                }
+                auto image = io::CreateImageFromFile(base_path + strpath);
+                if (image->HasData()) {
+                    img = image;
+                }
             }
         }
     };
@@ -254,7 +286,7 @@ bool ReadTriangleMeshUsingASSIMP(
 
         // Retrieve textures
         TextureImages maps;
-        LoadTextures(filename, mat, maps);
+        LoadTextures(filename, scene, mat, maps);
         mesh_material.albedo = maps.albedo;
         mesh_material.normalMap = maps.normal;
         mesh_material.ambientOcclusion = maps.ao;
@@ -413,7 +445,7 @@ bool ReadModelUsingAssimp(const std::string& filename,
 
         // Retrieve textures
         TextureImages maps;
-        LoadTextures(filename, mat, maps);
+        LoadTextures(filename, scene, mat, maps);
         o3d_mat.albedo_img = maps.albedo;
         o3d_mat.normal_img = maps.normal;
         o3d_mat.ao_img = maps.ao;

cpp/open3d/io/file_format/FileJPG.cpp

@@ -153,5 +153,56 @@ bool WriteImageToJPG(const std::string &filename,
     return true;
 }
 
+bool ReadJPGFromMemory(const unsigned char *image_data_ptr,
+                       size_t image_data_size,
+                       geometry::Image &image) {
+    struct jpeg_decompress_struct cinfo;
+    struct jpeg_error_mgr jerr;
+    JSAMPARRAY buffer;
+
+    cinfo.err = jpeg_std_error(&jerr);
+    jpeg_create_decompress(&cinfo);
+    jpeg_mem_src(&cinfo, image_data_ptr, image_data_size);
+    jpeg_read_header(&cinfo, TRUE);
+
+    // We only support two channel types: gray, and RGB.
+    int num_of_channels = 3;
+    int bytes_per_channel = 1;
+    switch (cinfo.jpeg_color_space) {
+        case JCS_RGB:
+        case JCS_YCbCr:
+            cinfo.out_color_space = JCS_RGB;
+            cinfo.out_color_components = 3;
+            num_of_channels = 3;
+            break;
+        case JCS_GRAYSCALE:
+            cinfo.jpeg_color_space = JCS_GRAYSCALE;
+            cinfo.out_color_components = 1;
+            num_of_channels = 1;
+            break;
+        case JCS_CMYK:
+        case JCS_YCCK:
+        default:
+            utility::LogWarning("Read JPG failed: color space not supported.");
+            jpeg_destroy_decompress(&cinfo);
+            return false;
+    }
+    jpeg_start_decompress(&cinfo);
+    image.Prepare(cinfo.output_width, cinfo.output_height, num_of_channels,
+                  bytes_per_channel);
+    int row_stride = cinfo.output_width * cinfo.output_components;
+    buffer = (*cinfo.mem->alloc_sarray)((j_common_ptr)&cinfo, JPOOL_IMAGE,
+                                        row_stride, 1);
+    uint8_t *pdata = image.data_.data();
+    while (cinfo.output_scanline < cinfo.output_height) {
+        jpeg_read_scanlines(&cinfo, buffer, 1);
+        memcpy(pdata, buffer[0], row_stride);
+        pdata += row_stride;
+    }
+    jpeg_finish_decompress(&cinfo);
+    jpeg_destroy_decompress(&cinfo);
+    return true;
+}
+
 }  // namespace io
 }  // namespace open3d

cpp/open3d/io/file_format/FilePNG.cpp

@@ -116,5 +116,35 @@ bool WriteImageToPNG(const std::string &filename,
     return true;
 }
 
+bool ReadPNGFromMemory(const unsigned char *image_data_ptr,
+                       size_t image_data_size,
+                       geometry::Image &image) {
+    png_image pngimage;
+    memset(&pngimage, 0, sizeof(pngimage));
+    pngimage.version = PNG_IMAGE_VERSION;
+    if (png_image_begin_read_from_memory(&pngimage, image_data_ptr,
+                                         image_data_size) == 0) {
+        utility::LogWarning("Read PNG failed: unable to parse header.");
+        return false;
+    }
+
+    // Clear colormap flag if necessary to ensure libpng expands the colo
+    // indexed pixels to full color
+    if (pngimage.format & PNG_FORMAT_FLAG_COLORMAP) {
+        pngimage.format &= ~PNG_FORMAT_FLAG_COLORMAP;
+    }
+
+    image.Prepare(pngimage.width, pngimage.height,
+                  PNG_IMAGE_SAMPLE_CHANNELS(pngimage.format),
+                  PNG_IMAGE_SAMPLE_COMPONENT_SIZE(pngimage.format));
+
+    if (png_image_finish_read(&pngimage, NULL, image.data_.data(), 0, NULL) ==
+        0) {
+        utility::LogWarning("PNG error: {}", pngimage.message);
+        return false;
+    }
+    return true;
+}
+
 }  // namespace io
 }  // namespace open3d

cpp/pybind/data/data.cpp

@@ -636,6 +636,36 @@ void pybind_helmet(py::module& m) {
     docstring::ClassMethodDocInject(m, "FlightHelmetModel", "path_map");
 }
 
+void pybind_avocado(py::module& m) {
+    // open3d.data.AvocadoModel
+    py::class_<AvocadoModel, PySingleDownloadDataset<AvocadoModel>,
+               std::shared_ptr<AvocadoModel>, SingleDownloadDataset>
+            avocado(m, "AvocadoModel",
+                    "Data class for `AvocadoModel` contains a avocado model "
+                    "file, "
+                    "along with material and PNG format embedded textures.");
+    avocado.def(py::init<const std::string&>(), "data_root"_a = "")
+            .def_property_readonly("path", &AvocadoModel::GetPath,
+                                   "Path to the `AvocadoModel.glb` file.");
+    docstring::ClassMethodDocInject(m, "AvocadoModel", "path");
+}
+
+void pybind_damaged_helmet(py::module& m) {
+    // open3d.data.DamagedHelmetModel
+    py::class_<DamagedHelmetModel, PySingleDownloadDataset<DamagedHelmetModel>,
+               std::shared_ptr<DamagedHelmetModel>, SingleDownloadDataset>
+            damaged_helmet(
+                    m, "DamagedHelmetModel",
+                    "Data class for `DamagedHelmetModel` contains a damaged "
+                    "helmet model file, "
+                    "along with material and JPG format embedded textures. ");
+    damaged_helmet.def(py::init<const std::string&>(), "data_root"_a = "")
+            .def_property_readonly(
+                    "path", &DamagedHelmetModel::GetPath,
+                    "Path to the `DamagedHelmetModel.glb` file.");
+    docstring::ClassMethodDocInject(m, "DamagedHelmetModel", "path");
+}
+
 void pybind_metal_texture(py::module& m) {
     // open3d.data.MetalTexture
     py::class_<MetalTexture, PySingleDownloadDataset<MetalTexture>,
@@ -986,6 +1016,8 @@ void pybind_data(py::module& m) {
     pybind_sword(m_submodule);
     pybind_crate(m_submodule);
     pybind_helmet(m_submodule);
+    pybind_avocado(m_submodule);
+    pybind_damaged_helmet(m_submodule);
     // Texture images for material.
     pybind_metal_texture(m_submodule);
     pybind_painted_plaster_texture(m_submodule);