diff --git a/cpp/open3d/geometry/TriangleMesh.h b/cpp/open3d/geometry/TriangleMesh.h
index 505c255ac26..8448cccbcc0 100644
--- a/cpp/open3d/geometry/TriangleMesh.h
+++ b/cpp/open3d/geometry/TriangleMesh.h
@@ -853,7 +853,7 @@ class TriangleMesh : public MeshBase {
         std::unordered_map<std::string, Image> additionalMaps;
     };
 
-    std::unordered_map<std::string, Material> materials_;
+    std::vector<std::pair<std::string, Material>> materials_;
 
     /// List of material ids.
     std::vector<int> triangle_material_ids_;
diff --git a/cpp/open3d/io/file_format/FileASSIMP.cpp b/cpp/open3d/io/file_format/FileASSIMP.cpp
index dec40259566..8604124dbed 100644
--- a/cpp/open3d/io/file_format/FileASSIMP.cpp
+++ b/cpp/open3d/io/file_format/FileASSIMP.cpp
@@ -237,12 +237,13 @@ bool ReadTriangleMeshUsingASSIMP(
     }
 
     // Now load the materials
+    mesh.materials_.resize(scene->mNumMaterials);
     for (size_t i = 0; i < scene->mNumMaterials; ++i) {
         auto* mat = scene->mMaterials[i];
 
-        // create material structure to match this name
-        auto& mesh_material =
-                mesh.materials_[std::string(mat->GetName().C_Str())];
+        // Set the material structure to match this name
+        auto& mesh_material = mesh.materials_[i].second;
+        mesh.materials_[i].first = mat->GetName().C_Str();
 
         using MaterialParameter =
                 geometry::TriangleMesh::Material::MaterialParameter;
@@ -277,9 +278,9 @@ bool ReadTriangleMeshUsingASSIMP(
 
         // For legacy visualization support
         if (mesh_material.albedo) {
-            mesh.textures_.push_back(*mesh_material.albedo->FlipVertical());
+            mesh.textures_.emplace_back(*mesh_material.albedo->FlipVertical());
         } else {
-            mesh.textures_.push_back(geometry::Image());
+            mesh.textures_.emplace_back();
         }
     }
 
diff --git a/cpp/open3d/io/file_format/FileOBJ.cpp b/cpp/open3d/io/file_format/FileOBJ.cpp
index 3123d8ad05a..a06b30a97fb 100644
--- a/cpp/open3d/io/file_format/FileOBJ.cpp
+++ b/cpp/open3d/io/file_format/FileOBJ.cpp
@@ -140,8 +140,11 @@ bool ReadTriangleMeshFromOBJ(const std::string& filename,
     using MaterialParameter =
             geometry::TriangleMesh::Material::MaterialParameter;
 
-    for (auto& material : materials) {
-        auto& meshMaterial = mesh.materials_[material.name];
+    mesh.materials_.resize(materials.size());
+    for (std::size_t i = 0; i < materials.size(); ++i) {
+        auto& material = materials[i];
+        mesh.materials_[i].first = material.name;
+        auto& meshMaterial = mesh.materials_[i].second;
 
         meshMaterial.baseColor = MaterialParameter::CreateRGB(
                 material.diffuse[0], material.diffuse[1], material.diffuse[2]);
diff --git a/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp b/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp
index 5dc25714392..55e85b2d578 100644
--- a/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp
+++ b/cpp/open3d/visualization/visualizer/O3DVisualizer.cpp
@@ -941,11 +941,24 @@ struct O3DVisualizer::Impl {
 
             // Finally assign material properties if geometry is a triangle mesh
             if (tmesh && tmesh->materials_.size() > 0) {
-                // Only a single material is supported for TriangleMesh so we
-                // just grab the first one we find. Users should be using
-                // TriangleMeshModel if they have a model with multiple
-                // materials.
-                auto &mesh_material = tmesh->materials_.begin()->second;
+                std::size_t material_index;
+                if (tmesh->HasTriangleMaterialIds()) {
+                    auto minmax_it = std::minmax_element(
+                            tmesh->triangle_material_ids_.begin(),
+                            tmesh->triangle_material_ids_.end());
+                    if (*minmax_it.first != *minmax_it.second) {
+                        utility::LogWarning(
+                                "Only a single material is "
+                                "supported for TriangleMesh visualization, "
+                                "only the first referenced material will be "
+                                "used. Use TriangleMeshModel if more than one "
+                                "material is required.");
+                    }
+                    material_index = *minmax_it.first;
+                } else {
+                    material_index = 0;
+                }
+                auto &mesh_material = tmesh->materials_[material_index].second;
                 mat.base_color = {mesh_material.baseColor.r(),
                                   mesh_material.baseColor.g(),
                                   mesh_material.baseColor.b(),