From 721dc99ea93894170e2db11738cb21fe2318a021 Mon Sep 17 00:00:00 2001
From: Leonard Hecker <lhecker@microsoft.com>
Date: Fri, 30 Jun 2023 16:23:52 +0200
Subject: [PATCH] AtlasEngine: Harden against empty target sizes

---
 src/renderer/atlas/AtlasEngine.api.cpp | 21 +++++++++++++--------
 src/renderer/atlas/BackendD3D.cpp      |  5 +++--
 src/renderer/atlas/common.h            |  4 ++--
 3 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/src/renderer/atlas/AtlasEngine.api.cpp b/src/renderer/atlas/AtlasEngine.api.cpp
index 817d94e5929..2bf0be6d71f 100644
--- a/src/renderer/atlas/AtlasEngine.api.cpp
+++ b/src/renderer/atlas/AtlasEngine.api.cpp
@@ -420,14 +420,19 @@ void AtlasEngine::SetWarningCallback(std::function<void(HRESULT)> pfn) noexcept
 
 [[nodiscard]] HRESULT AtlasEngine::SetWindowSize(const til::size pixels) noexcept
 {
-    u16x2 newSize;
-    RETURN_IF_FAILED(vec2_narrow(pixels.width, pixels.height, newSize));
-
-    // At the time of writing:
-    // When Win+D is pressed, `TriggerRedrawCursor` is called and a render pass is initiated.
-    // As conhost is in the background, GetClientRect will return {0,0} and we'll get called with {0,0}.
-    // This isn't a valid value for _api.sizeInPixel and would crash _recreateSizeDependentResources().
-    if (_api.s->targetSize != newSize && newSize != u16x2{})
+    // When Win+D is pressed, `GetClientRect` returns {0,0}.
+    // There's probably more situations in which our callers may pass us invalid data.
+    if (!pixels)
+    {
+        return S_OK;
+    }
+
+    const u16x2 newSize{
+        gsl::narrow_cast<u16>(min<til::CoordType>(pixels.width, u16max)),
+        gsl::narrow_cast<u16>(min<til::CoordType>(pixels.height, u16max)),
+    };
+
+    if (_api.s->targetSize != newSize)
     {
         _api.s.write()->targetSize = newSize;
     }
diff --git a/src/renderer/atlas/BackendD3D.cpp b/src/renderer/atlas/BackendD3D.cpp
index fb549acaf7b..056f2f6bbdc 100644
--- a/src/renderer/atlas/BackendD3D.cpp
+++ b/src/renderer/atlas/BackendD3D.cpp
@@ -693,7 +693,6 @@ void BackendD3D::_resetGlyphAtlas(const RenderingPayload& p)
 
     const auto minAreaByFont = cellArea * 95; // Covers all printable ASCII characters
     const auto minAreaByGrowth = static_cast<u32>(_rectPacker.width) * _rectPacker.height * 2;
-    const auto min = std::max(minArea, std::max(minAreaByFont, minAreaByGrowth));
 
     // It's hard to say what the max. size of the cache should be. Optimally I think we should use as much
     // memory as is available, but the rendering code in this project is a big mess and so integrating
@@ -702,7 +701,9 @@ void BackendD3D::_resetGlyphAtlas(const RenderingPayload& p)
     // we're locked into a state, where on every render pass we're starting with a half full atlas, drawing once,
     // filling it with the remaining half and drawing again, requiring two rendering passes on each frame.
     const auto maxAreaByFont = targetArea + targetArea / 4;
-    const auto area = std::min(maxArea, std::min(maxAreaByFont, min));
+
+    auto area = std::min(maxAreaByFont, std::max(minAreaByFont, minAreaByGrowth));
+    area = clamp(area, minArea, maxArea);
 
     // This block of code calculates the size of a power-of-2 texture that has an area larger than the given `area`.
     // For instance, for an area of 985x1946 = 1916810 it would result in a u/v of 2048x1024 (area = 2097152).
diff --git a/src/renderer/atlas/common.h b/src/renderer/atlas/common.h
index c0b69b808ec..2b752de8a5e 100644
--- a/src/renderer/atlas/common.h
+++ b/src/renderer/atlas/common.h
@@ -387,8 +387,8 @@ namespace Microsoft::Console::Render::Atlas
         til::generational<FontSettings> font;
         til::generational<CursorSettings> cursor;
         til::generational<MiscellaneousSettings> misc;
-        u16x2 targetSize{};
-        u16x2 cellCount{};
+        u16x2 targetSize{ 1, 1 };
+        u16x2 cellCount{ 1, 1 };
     };
 
     using GenerationalSettings = til::generational<Settings>;