From ed4450aa8c3319037a3b82e77bafd0049a13c8cb Mon Sep 17 00:00:00 2001 From: MeltyPlayer Date: Thu, 12 Jan 2023 20:52:41 -0600 Subject: [PATCH 1/9] Updated the image path used in the textured scene creation method so that it's more robust. Revert "Set up platforms for x86 and x64." This reverts commit 2ab2abd8aa27c71623e586ab4cf36bc6ba38b97d. Set up platforms for x86 and x64. --- .../Schema2/Authoring/BasicSceneCreationTests.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/SharpGLTF.Core.Tests/Schema2/Authoring/BasicSceneCreationTests.cs b/tests/SharpGLTF.Core.Tests/Schema2/Authoring/BasicSceneCreationTests.cs index 13eb04bb..b8b3fd9f 100644 --- a/tests/SharpGLTF.Core.Tests/Schema2/Authoring/BasicSceneCreationTests.cs +++ b/tests/SharpGLTF.Core.Tests/Schema2/Authoring/BasicSceneCreationTests.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.IO; using System.Numerics; using NUnit.Framework; @@ -117,7 +118,14 @@ public void CreateSceneWithTexturedTriangle() TestContext.CurrentContext.AttachGltfValidatorLinks(); // we'll use our icon as the source texture - var imagePath = System.IO.Path.Combine(TestContext.CurrentContext.WorkDirectory, "..\\..\\..\\..\\..\\build\\Icons\\glTF2Sharp.png"); + var workDirectory = new DirectoryInfo(TestContext.CurrentContext.WorkDirectory); + var testDirectory = workDirectory; + do { + testDirectory = testDirectory.Parent; + } while (testDirectory.Name != "tests"); + var sharpGltfDirectory = testDirectory.Parent; + + var imagePath = Path.Combine(sharpGltfDirectory.FullName, "build\\Icons\\glTF2Sharp.png"); // create a basic scene var model = ModelRoot.CreateModel(); From 80d5c3aa8550a91d2bceb373734358833049e0f3 Mon Sep 17 00:00:00 2001 From: MeltyPlayer Date: Thu, 12 Jan 2023 23:27:00 -0600 Subject: [PATCH 2/9] Further optimized WavefrontWriter's memory usage by having it write directly to the output FileStream rather than first to a MemoryStream and then copying. Optimized some redundant memory allocations in the WavefrontWriter class to avoid OutOfMemory exceptions in 32-bit mode, which fixes some of the existing tests. Made some more memory optimizations in the WavefrontWriter to limit how many objects are created and stored in memory at once. --- src/SharpGLTF.Toolkit/IO/WavefrontWriter.cs | 129 ++++++++++---------- 1 file changed, 67 insertions(+), 62 deletions(-) diff --git a/src/SharpGLTF.Toolkit/IO/WavefrontWriter.cs b/src/SharpGLTF.Toolkit/IO/WavefrontWriter.cs index f14bf030..f7d7f3ab 100644 --- a/src/SharpGLTF.Toolkit/IO/WavefrontWriter.cs +++ b/src/SharpGLTF.Toolkit/IO/WavefrontWriter.cs @@ -1,10 +1,12 @@ using System; using System.Collections; using System.Collections.Generic; +using System.IO; using System.Linq; using System.Numerics; using System.Text; +using SharpGLTF.Memory; using SharpGLTF.Schema2; using static System.FormattableString; @@ -55,14 +57,13 @@ public void WriteFiles(string filePath) { Guard.NotNullOrEmpty(filePath, nameof(filePath)); - var files = GetFiles(System.IO.Path.GetFileNameWithoutExtension(filePath)); - var dir = System.IO.Path.GetDirectoryName(filePath); - foreach (var f in files) + foreach (var fileNameAndGenerator in _GetFileGenerators(System.IO.Path.GetFileNameWithoutExtension(filePath))) { - var fpath = System.IO.Path.Combine(dir, f.Key); - System.IO.File.WriteAllBytes(fpath, f.Value.ToArray()); + var fpath = System.IO.Path.Combine(dir, fileNameAndGenerator.Key); + using var fs = File.OpenWrite(fpath); + fileNameAndGenerator.Value(fs); } } @@ -81,17 +82,33 @@ public IReadOnlyDictionary GetFiles(string baseName) Guard.IsFalse(baseName.Any(c => char.IsWhiteSpace(c)), nameof(baseName), "Whitespace characters not allowed in filename"); var files = new Dictionary(); + foreach (var fileNameAndGenerator in _GetFileGenerators(baseName)) + { + using var mem = new MemoryStream(); + fileNameAndGenerator.Value(mem); - var materials = _WriteMaterials(files, baseName, _Mesh.Primitives.Select(item => item.Material)); - - var geocontent = _GetGeometryContent(materials, baseName + ".mtl"); + mem.TryGetBuffer(out var bytes); - _WriteTextContent(files, baseName + ".obj", geocontent); + files[fileNameAndGenerator.Key] = bytes; + } return files; } - private static IReadOnlyDictionary _WriteMaterials(IDictionary files, string baseName, IEnumerable materials) + private IReadOnlyDictionary> _GetFileGenerators(string baseName) + { + Guard.IsFalse(baseName.Any(c => char.IsWhiteSpace(c)), nameof(baseName), "Whitespace characters not allowed in filename"); + + var fileGenerators = new Dictionary>(); + + var materials = _GetMaterialFileGenerator(fileGenerators, baseName, _Mesh.Primitives.Select(item => item.Material)); + + fileGenerators[baseName + ".obj"] = fs => _GetGeometryContent(new StreamWriter(fs), materials, baseName + ".mtl"); + + return fileGenerators; + } + + private static IReadOnlyDictionary _GetMaterialFileGenerator(IDictionary> fileGenerators, string baseName, IEnumerable materials) { // write all image files var images = materials @@ -101,76 +118,81 @@ private static IReadOnlyDictionary _WriteMaterials(IDictionary bool firstImg = true; + var imageNameByImage = new Dictionary(); foreach (var img in images) { var imgName = firstImg ? $"{baseName}.{img.FileExtension}" - : $"{baseName}_{files.Count}.{img.FileExtension}"; + : $"{baseName}_{fileGenerators.Count}.{img.FileExtension}"; - files[imgName] = new BYTES(img.Content.ToArray()); + fileGenerators[imgName] = fs => { + var bytes = img.Content.ToArray(); + fs.Write(bytes, 0, bytes.Length); + }; firstImg = false; + + imageNameByImage[img] = imgName; } // write materials var mmap = new Dictionary(); - - var sb = new StringBuilder(); - - foreach (var m in materials) + foreach (var m in materials) { mmap[m] = $"Material_{mmap.Count}"; - - sb.AppendLine($"newmtl {mmap[m]}"); - sb.AppendLine("illum 2"); - sb.AppendLine(Invariant($"Ka {m.DiffuseColor.X} {m.DiffuseColor.Y} {m.DiffuseColor.Z}")); - sb.AppendLine(Invariant($"Kd {m.DiffuseColor.X} {m.DiffuseColor.Y} {m.DiffuseColor.Z}")); - sb.AppendLine(Invariant($"Ks {m.SpecularColor.X} {m.SpecularColor.Y} {m.SpecularColor.Z}")); - - if (m.DiffuseTexture.IsValid) - { - var imgName = files.FirstOrDefault(kvp => new Memory.MemoryImage(kvp.Value) == m.DiffuseTexture ).Key; - sb.AppendLine($"map_Kd {imgName}"); - } - - sb.AppendLine(); } // write material library - _WriteTextContent(files, baseName + ".mtl", sb); + fileGenerators[baseName + ".mtl"] = fs => + { + var sw = new StreamWriter(fs); + foreach (var m in materials) + { + sw.WriteLine($"newmtl {mmap[m]}"); + sw.WriteLine("illum 2"); + sw.WriteLine(Invariant($"Ka {m.DiffuseColor.X} {m.DiffuseColor.Y} {m.DiffuseColor.Z}")); + sw.WriteLine(Invariant($"Kd {m.DiffuseColor.X} {m.DiffuseColor.Y} {m.DiffuseColor.Z}")); + sw.WriteLine(Invariant($"Ks {m.SpecularColor.X} {m.SpecularColor.Y} {m.SpecularColor.Z}")); + + if (m.DiffuseTexture.IsValid) { + var imgName = imageNameByImage[m.DiffuseTexture]; + sw.WriteLine($"map_Kd {imgName}"); + } + + sw.WriteLine(); + } + }; return mmap; } - private StringBuilder _GetGeometryContent(IReadOnlyDictionary materials, string mtlLib) + private void _GetGeometryContent(StreamWriter sw, IReadOnlyDictionary materials, string mtlLib) { - var sb = new StringBuilder(); - - sb.AppendLine($"mtllib {mtlLib}"); + sw.WriteLine($"mtllib {mtlLib}"); - sb.AppendLine(); + sw.WriteLine(); foreach (var p in _Mesh.Primitives) { foreach (var v in p.Vertices) { var pos = v.Position; - sb.AppendLine(Invariant($"v {pos.X} {pos.Y} {pos.Z}")); + sw.WriteLine(Invariant($"v {pos.X} {pos.Y} {pos.Z}")); } } - sb.AppendLine(); + sw.WriteLine(); foreach (var p in _Mesh.Primitives) { foreach (var v in p.Vertices) { var nrm = v.Geometry.Normal; - sb.AppendLine(Invariant($"vn {nrm.X} {nrm.Y} {nrm.Z}")); + sw.WriteLine(Invariant($"vn {nrm.X} {nrm.Y} {nrm.Z}")); } } - sb.AppendLine(); + sw.WriteLine(); foreach (var p in _Mesh.Primitives) { @@ -179,13 +201,13 @@ private StringBuilder _GetGeometryContent(IReadOnlyDictionary var uv = v.Material.TexCoord; uv.Y = 1 - uv.Y; - sb.AppendLine(Invariant($"vt {uv.X} {uv.Y}")); + sw.WriteLine(Invariant($"vt {uv.X} {uv.Y}")); } } - sb.AppendLine(); + sw.WriteLine(); - sb.AppendLine("g default"); + sw.WriteLine("g default"); var baseVertexIndex = 1; @@ -193,7 +215,7 @@ private StringBuilder _GetGeometryContent(IReadOnlyDictionary { var mtl = materials[p.Material]; - sb.AppendLine($"usemtl {mtl}"); + sw.WriteLine($"usemtl {mtl}"); foreach (var t in p.Triangles) { @@ -201,28 +223,11 @@ private StringBuilder _GetGeometryContent(IReadOnlyDictionary var b = t.B + baseVertexIndex; var c = t.C + baseVertexIndex; - sb.AppendLine(Invariant($"f {a}/{a}/{a} {b}/{b}/{b} {c}/{c}/{c}")); + sw.WriteLine(Invariant($"f {a}/{a}/{a} {b}/{b}/{b} {c}/{c}/{c}")); } baseVertexIndex += p.Vertices.Count; } - - return sb; - } - - private static void _WriteTextContent(IDictionary files, string fileName, StringBuilder sb) - { - using (var mem = new System.IO.MemoryStream()) - { - using (var tex = new System.IO.StreamWriter(mem)) - { - tex.Write(sb.ToString()); - } - - mem.TryGetBuffer(out BYTES content); - - files[fileName] = content; - } } #endregion From 3f9fb44e112261fdeb64471a7b38a429726f08c8 Mon Sep 17 00:00:00 2001 From: MeltyPlayer Date: Fri, 13 Jan 2023 17:04:33 -0600 Subject: [PATCH 3/9] Initialized the buffer capacity in MergeBuffers() to fix OutOfMemory issues, and added a test covering this. --- src/SharpGLTF.Core/Schema2/gltf.Buffer.cs | 2 +- src/SharpGLTF.Core/Schema2/gltf.BufferView.cs | 5 +- .../Scenes/SceneBuilderTests.cs | 94 +++++++++++++++++++ .../SharpGLTF.Toolkit.Tests.csproj | 3 +- 4 files changed, 100 insertions(+), 4 deletions(-) diff --git a/src/SharpGLTF.Core/Schema2/gltf.Buffer.cs b/src/SharpGLTF.Core/Schema2/gltf.Buffer.cs index 417cd2ef..a9b88683 100644 --- a/src/SharpGLTF.Core/Schema2/gltf.Buffer.cs +++ b/src/SharpGLTF.Core/Schema2/gltf.Buffer.cs @@ -220,7 +220,7 @@ public void MergeBuffers() // begin merge - var sbbuilder = new _StaticBufferBuilder(0); + var sbbuilder = new _StaticBufferBuilder(0, (int) totalLen); foreach (var bv in views) bv._IsolateBufferMemory(sbbuilder); diff --git a/src/SharpGLTF.Core/Schema2/gltf.BufferView.cs b/src/SharpGLTF.Core/Schema2/gltf.BufferView.cs index d09f86da..db3d1f55 100644 --- a/src/SharpGLTF.Core/Schema2/gltf.BufferView.cs +++ b/src/SharpGLTF.Core/Schema2/gltf.BufferView.cs @@ -347,9 +347,10 @@ sealed class _StaticBufferBuilder { #region lifecycle - public _StaticBufferBuilder(int bufferIndex) + public _StaticBufferBuilder(int bufferIndex, int initialCapacity = 0) { _BufferIndex = bufferIndex; + _Data = new List(initialCapacity); } #endregion @@ -360,7 +361,7 @@ public _StaticBufferBuilder(int bufferIndex) private readonly int _BufferIndex; // accumulated data - private readonly List _Data = new List(); + private readonly List _Data; #endregion diff --git a/tests/SharpGLTF.Toolkit.Tests/Scenes/SceneBuilderTests.cs b/tests/SharpGLTF.Toolkit.Tests/Scenes/SceneBuilderTests.cs index bab347d8..5affc1b4 100644 --- a/tests/SharpGLTF.Toolkit.Tests/Scenes/SceneBuilderTests.cs +++ b/tests/SharpGLTF.Toolkit.Tests/Scenes/SceneBuilderTests.cs @@ -991,5 +991,99 @@ public void TestSceneAddition() gltf.AttachToCurrentTest("Three cubes.glb"); } + [Test] + public void TestHugeSceneViaBuilders() + { + // create a scene + + var mesh = new MeshBuilder(); + + var gridSize = 1000; + + // generate texture + var diffuseImageSize = 2 * gridSize; + var occlusionImageSize = gridSize; + + MemoryImage diffuseImage, occlusionImage; + { + var img = new Bitmap(diffuseImageSize, diffuseImageSize, PixelFormat.Format32bppArgb); + + using var mem = new MemoryStream(); + img.Save(mem, ImageFormat.Png); + + mem.TryGetBuffer(out var bytes); + diffuseImage = new MemoryImage(bytes); + } + { + var img = new Bitmap(occlusionImageSize, occlusionImageSize, PixelFormat.Format32bppArgb); + + using var mem = new MemoryStream(); + img.Save(mem, ImageFormat.Png); + + mem.TryGetBuffer(out var bytes); + occlusionImage = new MemoryImage(bytes); + } + + var material = MaterialBuilder.CreateDefault().WithSpecularGlossinessShader(); + material.UseChannel(KnownChannel.Diffuse) + .UseTexture() + .WithPrimaryImage(ImageBuilder.From(diffuseImage)) + .WithCoordinateSet(0); + material.UseChannel(KnownChannel.Occlusion) + .UseTexture() + .WithPrimaryImage(ImageBuilder.From(occlusionImage)) + .WithCoordinateSet(0); + + // generate heightmap + + for (var y = 0; y < gridSize; ++y) + { + for (var x = 0; x < gridSize; ++x) + { + var vertices = new (float X, float Y)[] + { + (x, y), + (x + 1, y), + (x, y + 1), + (x + 1, y + 1) + }.Select(pos => VertexBuilder + .Create(new Vector3(pos.X, pos.Y, 0), new Vector3(x, y, 0)) + .WithMaterial(new Vector4(pos.X / gridSize, pos.Y / gridSize, 0, 1), + new Vector4(0, pos.X / gridSize, pos.Y / gridSize, 1), + new Vector2(pos.X / gridSize, pos.Y / gridSize), + new Vector2(pos.X / gridSize, pos.Y / gridSize)) + .WithSkinning(SparseWeight8.Create((0, 1)))) + .ToArray(); + + mesh.UsePrimitive(material).AddTriangle(vertices[0], vertices[1], vertices[2]); + mesh.UsePrimitive(material).AddTriangle(vertices[1], vertices[2], vertices[3]); + } + } + + var scene = new SceneBuilder(); + scene.AddSkinnedMesh(mesh, Matrix4x4.Identity, new NodeBuilder()); + + // convert to gltf + + var gltf = scene.ToGltf2(); + + Assert.AreEqual(1, gltf.LogicalMeshes.Count); + + var outFiles = new[] + { + "huge_scene.glb", + "huge_scene.gltf", + "huge_scene.obj", + }; + + foreach (var outFile in outFiles) + { + gltf.AttachToCurrentTest(outFile); + + GC.WaitForPendingFinalizers(); + GC.Collect(); + GC.WaitForFullGCComplete(); + } + } } } diff --git a/tests/SharpGLTF.Toolkit.Tests/SharpGLTF.Toolkit.Tests.csproj b/tests/SharpGLTF.Toolkit.Tests/SharpGLTF.Toolkit.Tests.csproj index cb396f65..1a542b3c 100644 --- a/tests/SharpGLTF.Toolkit.Tests/SharpGLTF.Toolkit.Tests.csproj +++ b/tests/SharpGLTF.Toolkit.Tests/SharpGLTF.Toolkit.Tests.csproj @@ -26,7 +26,8 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + + \ No newline at end of file From 1e0f91aaa737b42b4f2eb786021e6f37854090a3 Mon Sep 17 00:00:00 2001 From: MeltyPlayer Date: Sat, 14 Jan 2023 18:01:23 -0600 Subject: [PATCH 4/9] Revert "Updated the image path used in the textured scene creation method so that it's more robust." This reverts commit ed4450aa8c3319037a3b82e77bafd0049a13c8cb. --- .../Schema2/Authoring/BasicSceneCreationTests.cs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/SharpGLTF.Core.Tests/Schema2/Authoring/BasicSceneCreationTests.cs b/tests/SharpGLTF.Core.Tests/Schema2/Authoring/BasicSceneCreationTests.cs index b8b3fd9f..13eb04bb 100644 --- a/tests/SharpGLTF.Core.Tests/Schema2/Authoring/BasicSceneCreationTests.cs +++ b/tests/SharpGLTF.Core.Tests/Schema2/Authoring/BasicSceneCreationTests.cs @@ -1,5 +1,4 @@ using System.Collections.Generic; -using System.IO; using System.Numerics; using NUnit.Framework; @@ -118,14 +117,7 @@ public void CreateSceneWithTexturedTriangle() TestContext.CurrentContext.AttachGltfValidatorLinks(); // we'll use our icon as the source texture - var workDirectory = new DirectoryInfo(TestContext.CurrentContext.WorkDirectory); - var testDirectory = workDirectory; - do { - testDirectory = testDirectory.Parent; - } while (testDirectory.Name != "tests"); - var sharpGltfDirectory = testDirectory.Parent; - - var imagePath = Path.Combine(sharpGltfDirectory.FullName, "build\\Icons\\glTF2Sharp.png"); + var imagePath = System.IO.Path.Combine(TestContext.CurrentContext.WorkDirectory, "..\\..\\..\\..\\..\\build\\Icons\\glTF2Sharp.png"); // create a basic scene var model = ModelRoot.CreateModel(); From 4a380e0428ee876ebc4c6255ce8f523a3a552453 Mon Sep 17 00:00:00 2001 From: MeltyPlayer Date: Sat, 14 Jan 2023 18:02:03 -0600 Subject: [PATCH 5/9] Revert "Further optimized WavefrontWriter's memory usage by having it write directly to the output FileStream rather than first to a MemoryStream and then copying." This reverts commit 80d5c3aa8550a91d2bceb373734358833049e0f3. --- src/SharpGLTF.Toolkit/IO/WavefrontWriter.cs | 129 ++++++++++---------- 1 file changed, 62 insertions(+), 67 deletions(-) diff --git a/src/SharpGLTF.Toolkit/IO/WavefrontWriter.cs b/src/SharpGLTF.Toolkit/IO/WavefrontWriter.cs index f7d7f3ab..f14bf030 100644 --- a/src/SharpGLTF.Toolkit/IO/WavefrontWriter.cs +++ b/src/SharpGLTF.Toolkit/IO/WavefrontWriter.cs @@ -1,12 +1,10 @@ using System; using System.Collections; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Numerics; using System.Text; -using SharpGLTF.Memory; using SharpGLTF.Schema2; using static System.FormattableString; @@ -57,13 +55,14 @@ public void WriteFiles(string filePath) { Guard.NotNullOrEmpty(filePath, nameof(filePath)); + var files = GetFiles(System.IO.Path.GetFileNameWithoutExtension(filePath)); + var dir = System.IO.Path.GetDirectoryName(filePath); - foreach (var fileNameAndGenerator in _GetFileGenerators(System.IO.Path.GetFileNameWithoutExtension(filePath))) + foreach (var f in files) { - var fpath = System.IO.Path.Combine(dir, fileNameAndGenerator.Key); - using var fs = File.OpenWrite(fpath); - fileNameAndGenerator.Value(fs); + var fpath = System.IO.Path.Combine(dir, f.Key); + System.IO.File.WriteAllBytes(fpath, f.Value.ToArray()); } } @@ -82,33 +81,17 @@ public IReadOnlyDictionary GetFiles(string baseName) Guard.IsFalse(baseName.Any(c => char.IsWhiteSpace(c)), nameof(baseName), "Whitespace characters not allowed in filename"); var files = new Dictionary(); - foreach (var fileNameAndGenerator in _GetFileGenerators(baseName)) - { - using var mem = new MemoryStream(); - fileNameAndGenerator.Value(mem); - mem.TryGetBuffer(out var bytes); + var materials = _WriteMaterials(files, baseName, _Mesh.Primitives.Select(item => item.Material)); - files[fileNameAndGenerator.Key] = bytes; - } - - return files; - } - - private IReadOnlyDictionary> _GetFileGenerators(string baseName) - { - Guard.IsFalse(baseName.Any(c => char.IsWhiteSpace(c)), nameof(baseName), "Whitespace characters not allowed in filename"); + var geocontent = _GetGeometryContent(materials, baseName + ".mtl"); - var fileGenerators = new Dictionary>(); + _WriteTextContent(files, baseName + ".obj", geocontent); - var materials = _GetMaterialFileGenerator(fileGenerators, baseName, _Mesh.Primitives.Select(item => item.Material)); - - fileGenerators[baseName + ".obj"] = fs => _GetGeometryContent(new StreamWriter(fs), materials, baseName + ".mtl"); - - return fileGenerators; + return files; } - private static IReadOnlyDictionary _GetMaterialFileGenerator(IDictionary> fileGenerators, string baseName, IEnumerable materials) + private static IReadOnlyDictionary _WriteMaterials(IDictionary files, string baseName, IEnumerable materials) { // write all image files var images = materials @@ -118,81 +101,76 @@ private static IReadOnlyDictionary _GetMaterialFileGenerator(I bool firstImg = true; - var imageNameByImage = new Dictionary(); foreach (var img in images) { var imgName = firstImg ? $"{baseName}.{img.FileExtension}" - : $"{baseName}_{fileGenerators.Count}.{img.FileExtension}"; + : $"{baseName}_{files.Count}.{img.FileExtension}"; - fileGenerators[imgName] = fs => { - var bytes = img.Content.ToArray(); - fs.Write(bytes, 0, bytes.Length); - }; + files[imgName] = new BYTES(img.Content.ToArray()); firstImg = false; - - imageNameByImage[img] = imgName; } // write materials var mmap = new Dictionary(); - foreach (var m in materials) + + var sb = new StringBuilder(); + + foreach (var m in materials) { mmap[m] = $"Material_{mmap.Count}"; - } - // write material library - fileGenerators[baseName + ".mtl"] = fs => - { - var sw = new StreamWriter(fs); - foreach (var m in materials) + sb.AppendLine($"newmtl {mmap[m]}"); + sb.AppendLine("illum 2"); + sb.AppendLine(Invariant($"Ka {m.DiffuseColor.X} {m.DiffuseColor.Y} {m.DiffuseColor.Z}")); + sb.AppendLine(Invariant($"Kd {m.DiffuseColor.X} {m.DiffuseColor.Y} {m.DiffuseColor.Z}")); + sb.AppendLine(Invariant($"Ks {m.SpecularColor.X} {m.SpecularColor.Y} {m.SpecularColor.Z}")); + + if (m.DiffuseTexture.IsValid) { - sw.WriteLine($"newmtl {mmap[m]}"); - sw.WriteLine("illum 2"); - sw.WriteLine(Invariant($"Ka {m.DiffuseColor.X} {m.DiffuseColor.Y} {m.DiffuseColor.Z}")); - sw.WriteLine(Invariant($"Kd {m.DiffuseColor.X} {m.DiffuseColor.Y} {m.DiffuseColor.Z}")); - sw.WriteLine(Invariant($"Ks {m.SpecularColor.X} {m.SpecularColor.Y} {m.SpecularColor.Z}")); - - if (m.DiffuseTexture.IsValid) { - var imgName = imageNameByImage[m.DiffuseTexture]; - sw.WriteLine($"map_Kd {imgName}"); - } - - sw.WriteLine(); + var imgName = files.FirstOrDefault(kvp => new Memory.MemoryImage(kvp.Value) == m.DiffuseTexture ).Key; + sb.AppendLine($"map_Kd {imgName}"); } - }; + + sb.AppendLine(); + } + + // write material library + _WriteTextContent(files, baseName + ".mtl", sb); return mmap; } - private void _GetGeometryContent(StreamWriter sw, IReadOnlyDictionary materials, string mtlLib) + private StringBuilder _GetGeometryContent(IReadOnlyDictionary materials, string mtlLib) { - sw.WriteLine($"mtllib {mtlLib}"); + var sb = new StringBuilder(); + + sb.AppendLine($"mtllib {mtlLib}"); - sw.WriteLine(); + sb.AppendLine(); foreach (var p in _Mesh.Primitives) { foreach (var v in p.Vertices) { var pos = v.Position; - sw.WriteLine(Invariant($"v {pos.X} {pos.Y} {pos.Z}")); + sb.AppendLine(Invariant($"v {pos.X} {pos.Y} {pos.Z}")); } } - sw.WriteLine(); + sb.AppendLine(); foreach (var p in _Mesh.Primitives) { foreach (var v in p.Vertices) { var nrm = v.Geometry.Normal; - sw.WriteLine(Invariant($"vn {nrm.X} {nrm.Y} {nrm.Z}")); + sb.AppendLine(Invariant($"vn {nrm.X} {nrm.Y} {nrm.Z}")); } } - sw.WriteLine(); + sb.AppendLine(); foreach (var p in _Mesh.Primitives) { @@ -201,13 +179,13 @@ private void _GetGeometryContent(StreamWriter sw, IReadOnlyDictionary files, string fileName, StringBuilder sb) + { + using (var mem = new System.IO.MemoryStream()) + { + using (var tex = new System.IO.StreamWriter(mem)) + { + tex.Write(sb.ToString()); + } + + mem.TryGetBuffer(out BYTES content); + + files[fileName] = content; + } } #endregion From 3069248b548eb32d462cd78a8f70bfdb8f8fdaa3 Mon Sep 17 00:00:00 2001 From: MeltyPlayer Date: Sun, 15 Jan 2023 13:22:09 -0600 Subject: [PATCH 6/9] Moved the huge scene test from SceneBuilderTests.cs to MeltyPLayerTests.cs. --- .../MeltyPlayerTests.cs | 105 +++++++++++++++++- .../SharpGLTF.ThirdParty.Tests.csproj | 3 +- .../Scenes/SceneBuilderTests.cs | 94 ---------------- .../SharpGLTF.Toolkit.Tests.csproj | 1 - 4 files changed, 103 insertions(+), 100 deletions(-) diff --git a/tests/SharpGLTF.ThirdParty.Tests/MeltyPlayerTests.cs b/tests/SharpGLTF.ThirdParty.Tests/MeltyPlayerTests.cs index 67992dfa..15df3714 100644 --- a/tests/SharpGLTF.ThirdParty.Tests/MeltyPlayerTests.cs +++ b/tests/SharpGLTF.ThirdParty.Tests/MeltyPlayerTests.cs @@ -1,13 +1,18 @@ using System; -using System.Collections.Generic; +using System.Drawing; +using System.Drawing.Imaging; +using System.IO; using System.Linq; using System.Numerics; -using System.Text; -using System.Threading.Tasks; using NUnit.Framework; - +using SharpGLTF.Geometry; +using SharpGLTF.Geometry.VertexTypes; +using SharpGLTF.Materials; +using SharpGLTF.Memory; +using SharpGLTF.Scenes; using SharpGLTF.Schema2; +using SharpGLTF.Transforms; namespace SharpGLTF.ThirdParty { @@ -76,5 +81,97 @@ void addNodeMesh(string name, Matrix4x4 xform) Assert.AreEqual(4, satellites.Length); } + + [Test] + public void TestHugeSceneViaBuilders() + { + // create a scene + + var mesh = new MeshBuilder(); + + var gridSize = 1000; + + // generate texture + var diffuseImageSize = 2 * gridSize; + var occlusionImageSize = gridSize; + + MemoryImage diffuseImage, occlusionImage; + { + var img = new Bitmap(diffuseImageSize, diffuseImageSize, PixelFormat.Format32bppArgb); + + using var mem = new MemoryStream(); + img.Save(mem, ImageFormat.Png); + + mem.TryGetBuffer(out var bytes); + diffuseImage = new MemoryImage(bytes); + } + { + var img = new Bitmap(occlusionImageSize, occlusionImageSize, PixelFormat.Format32bppArgb); + + using var mem = new MemoryStream(); + img.Save(mem, ImageFormat.Png); + + mem.TryGetBuffer(out var bytes); + occlusionImage = new MemoryImage(bytes); + } + + var material = MaterialBuilder.CreateDefault().WithSpecularGlossinessShader(); + material.UseChannel(KnownChannel.Diffuse) + .UseTexture() + .WithPrimaryImage(ImageBuilder.From(diffuseImage)) + .WithCoordinateSet(0); + material.UseChannel(KnownChannel.Occlusion) + .UseTexture() + .WithPrimaryImage(ImageBuilder.From(occlusionImage)) + .WithCoordinateSet(0); + + // generate heightmap + + for (var y = 0; y < gridSize; ++y) { + for (var x = 0; x < gridSize; ++x) { + var vertices = new (float X, float Y)[] + { + (x, y), + (x + 1, y), + (x, y + 1), + (x + 1, y + 1) + }.Select(pos => VertexBuilder + .Create(new Vector3(pos.X, pos.Y, 0), new Vector3(x, y, 0)) + .WithMaterial(new Vector4(pos.X / gridSize, pos.Y / gridSize, 0, 1), + new Vector4(0, pos.X / gridSize, pos.Y / gridSize, 1), + new Vector2(pos.X / gridSize, pos.Y / gridSize), + new Vector2(pos.X / gridSize, pos.Y / gridSize)) + .WithSkinning(SparseWeight8.Create((0, 1)))) + .ToArray(); + + mesh.UsePrimitive(material).AddTriangle(vertices[0], vertices[1], vertices[2]); + mesh.UsePrimitive(material).AddTriangle(vertices[1], vertices[2], vertices[3]); + } + } + + var scene = new SceneBuilder(); + scene.AddSkinnedMesh(mesh, Matrix4x4.Identity, new NodeBuilder()); + + // convert to gltf + + var gltf = scene.ToGltf2(); + + Assert.AreEqual(1, gltf.LogicalMeshes.Count); + + var outFiles = new[] + { + "huge_scene.glb", + "huge_scene.gltf", + "huge_scene.obj", + }; + + foreach (var outFile in outFiles) { + gltf.AttachToCurrentTest(outFile); + + GC.WaitForPendingFinalizers(); + GC.Collect(); + GC.WaitForFullGCComplete(); + } + } } } diff --git a/tests/SharpGLTF.ThirdParty.Tests/SharpGLTF.ThirdParty.Tests.csproj b/tests/SharpGLTF.ThirdParty.Tests/SharpGLTF.ThirdParty.Tests.csproj index 0e73bfd5..b584e09d 100644 --- a/tests/SharpGLTF.ThirdParty.Tests/SharpGLTF.ThirdParty.Tests.csproj +++ b/tests/SharpGLTF.ThirdParty.Tests/SharpGLTF.ThirdParty.Tests.csproj @@ -18,7 +18,8 @@ - + + diff --git a/tests/SharpGLTF.Toolkit.Tests/Scenes/SceneBuilderTests.cs b/tests/SharpGLTF.Toolkit.Tests/Scenes/SceneBuilderTests.cs index 5affc1b4..bab347d8 100644 --- a/tests/SharpGLTF.Toolkit.Tests/Scenes/SceneBuilderTests.cs +++ b/tests/SharpGLTF.Toolkit.Tests/Scenes/SceneBuilderTests.cs @@ -991,99 +991,5 @@ public void TestSceneAddition() gltf.AttachToCurrentTest("Three cubes.glb"); } - [Test] - public void TestHugeSceneViaBuilders() - { - // create a scene - - var mesh = new MeshBuilder(); - - var gridSize = 1000; - - // generate texture - var diffuseImageSize = 2 * gridSize; - var occlusionImageSize = gridSize; - - MemoryImage diffuseImage, occlusionImage; - { - var img = new Bitmap(diffuseImageSize, diffuseImageSize, PixelFormat.Format32bppArgb); - - using var mem = new MemoryStream(); - img.Save(mem, ImageFormat.Png); - - mem.TryGetBuffer(out var bytes); - diffuseImage = new MemoryImage(bytes); - } - { - var img = new Bitmap(occlusionImageSize, occlusionImageSize, PixelFormat.Format32bppArgb); - - using var mem = new MemoryStream(); - img.Save(mem, ImageFormat.Png); - - mem.TryGetBuffer(out var bytes); - occlusionImage = new MemoryImage(bytes); - } - - var material = MaterialBuilder.CreateDefault().WithSpecularGlossinessShader(); - material.UseChannel(KnownChannel.Diffuse) - .UseTexture() - .WithPrimaryImage(ImageBuilder.From(diffuseImage)) - .WithCoordinateSet(0); - material.UseChannel(KnownChannel.Occlusion) - .UseTexture() - .WithPrimaryImage(ImageBuilder.From(occlusionImage)) - .WithCoordinateSet(0); - - // generate heightmap - - for (var y = 0; y < gridSize; ++y) - { - for (var x = 0; x < gridSize; ++x) - { - var vertices = new (float X, float Y)[] - { - (x, y), - (x + 1, y), - (x, y + 1), - (x + 1, y + 1) - }.Select(pos => VertexBuilder - .Create(new Vector3(pos.X, pos.Y, 0), new Vector3(x, y, 0)) - .WithMaterial(new Vector4(pos.X / gridSize, pos.Y / gridSize, 0, 1), - new Vector4(0, pos.X / gridSize, pos.Y / gridSize, 1), - new Vector2(pos.X / gridSize, pos.Y / gridSize), - new Vector2(pos.X / gridSize, pos.Y / gridSize)) - .WithSkinning(SparseWeight8.Create((0, 1)))) - .ToArray(); - - mesh.UsePrimitive(material).AddTriangle(vertices[0], vertices[1], vertices[2]); - mesh.UsePrimitive(material).AddTriangle(vertices[1], vertices[2], vertices[3]); - } - } - - var scene = new SceneBuilder(); - scene.AddSkinnedMesh(mesh, Matrix4x4.Identity, new NodeBuilder()); - - // convert to gltf - - var gltf = scene.ToGltf2(); - - Assert.AreEqual(1, gltf.LogicalMeshes.Count); - - var outFiles = new[] - { - "huge_scene.glb", - "huge_scene.gltf", - "huge_scene.obj", - }; - - foreach (var outFile in outFiles) - { - gltf.AttachToCurrentTest(outFile); - - GC.WaitForPendingFinalizers(); - GC.Collect(); - GC.WaitForFullGCComplete(); - } - } } } diff --git a/tests/SharpGLTF.Toolkit.Tests/SharpGLTF.Toolkit.Tests.csproj b/tests/SharpGLTF.Toolkit.Tests/SharpGLTF.Toolkit.Tests.csproj index 1a542b3c..26aacfca 100644 --- a/tests/SharpGLTF.Toolkit.Tests/SharpGLTF.Toolkit.Tests.csproj +++ b/tests/SharpGLTF.Toolkit.Tests/SharpGLTF.Toolkit.Tests.csproj @@ -27,7 +27,6 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - \ No newline at end of file From b881d631400e86055973258f134220d51ef8d0dc Mon Sep 17 00:00:00 2001 From: MeltyPlayer Date: Sun, 15 Jan 2023 13:23:16 -0600 Subject: [PATCH 7/9] Added some spaces back to the Toolkit Tests project file so that it's removed from the PR changes. --- tests/SharpGLTF.Toolkit.Tests/SharpGLTF.Toolkit.Tests.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/SharpGLTF.Toolkit.Tests/SharpGLTF.Toolkit.Tests.csproj b/tests/SharpGLTF.Toolkit.Tests/SharpGLTF.Toolkit.Tests.csproj index 26aacfca..88595969 100644 --- a/tests/SharpGLTF.Toolkit.Tests/SharpGLTF.Toolkit.Tests.csproj +++ b/tests/SharpGLTF.Toolkit.Tests/SharpGLTF.Toolkit.Tests.csproj @@ -26,7 +26,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + - \ No newline at end of file + From a23e44186330f034046a497036b20dcb6b1fc2a7 Mon Sep 17 00:00:00 2001 From: MeltyPlayer Date: Sun, 15 Jan 2023 13:24:56 -0600 Subject: [PATCH 8/9] Removed the trailing space from the Toolkit Tests file, should finally not be marked as a change anymore. --- tests/SharpGLTF.Toolkit.Tests/SharpGLTF.Toolkit.Tests.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/SharpGLTF.Toolkit.Tests/SharpGLTF.Toolkit.Tests.csproj b/tests/SharpGLTF.Toolkit.Tests/SharpGLTF.Toolkit.Tests.csproj index 88595969..cb396f65 100644 --- a/tests/SharpGLTF.Toolkit.Tests/SharpGLTF.Toolkit.Tests.csproj +++ b/tests/SharpGLTF.Toolkit.Tests/SharpGLTF.Toolkit.Tests.csproj @@ -29,4 +29,4 @@ - + \ No newline at end of file From f52b9f8be054b947f4fa63f4f05f46121a3cb0f0 Mon Sep 17 00:00:00 2001 From: MeltyPlayer Date: Sun, 15 Jan 2023 13:31:05 -0600 Subject: [PATCH 9/9] Increased reserved memory for the _StaticBufferBuilder, just to be safe. --- src/SharpGLTF.Core/Schema2/gltf.Buffer.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/SharpGLTF.Core/Schema2/gltf.Buffer.cs b/src/SharpGLTF.Core/Schema2/gltf.Buffer.cs index a9b88683..df4d0946 100644 --- a/src/SharpGLTF.Core/Schema2/gltf.Buffer.cs +++ b/src/SharpGLTF.Core/Schema2/gltf.Buffer.cs @@ -220,7 +220,8 @@ public void MergeBuffers() // begin merge - var sbbuilder = new _StaticBufferBuilder(0, (int) totalLen); + var reservedMemory = (int) (totalLen * 1.01); // increase by 1% + var sbbuilder = new _StaticBufferBuilder(0, reservedMemory); foreach (var bv in views) bv._IsolateBufferMemory(sbbuilder);