From 5d4e6ced4b91862b1a6982786b9339175b8caf76 Mon Sep 17 00:00:00 2001
From: "Dustin L. Howett (MSFT)" <duhowett@microsoft.com>
Date: Mon, 23 Sep 2019 15:06:47 -0700
Subject: [PATCH] Add some retry support to Renderer::PaintFrame (#2830)

If _PaintFrameForEngine returns E_PENDING, we'll give it another two
tries to get itself straight. If it continues to fail, we'll take down
the application.

We observed that the DX renderer was failing to present the swap chain
and failfast'ing when it did so; however, there are some errors from
which DXGI guidance suggests we try to recover. We'll now return
E_PENDING (and destroy our device resources) when we hit those errors.

Fixes #2265.

(cherry picked from commit 277acc3383ae3dace82c08ef46549b6eba654a73)
---
 src/renderer/base/renderer.cpp | 18 +++++++++++++++++-
 src/renderer/dx/DxRenderer.cpp | 22 +++++++++++++++++++---
 2 files changed, 36 insertions(+), 4 deletions(-)

diff --git a/src/renderer/base/renderer.cpp b/src/renderer/base/renderer.cpp
index 2dffc9a1550..e434c5d0725 100644
--- a/src/renderer/base/renderer.cpp
+++ b/src/renderer/base/renderer.cpp
@@ -10,6 +10,8 @@
 using namespace Microsoft::Console::Render;
 using namespace Microsoft::Console::Types;
 
+static constexpr auto maxRetriesForRenderEngine = 3;
+
 // Routine Description:
 // - Creates a new renderer controller for a console.
 // Arguments:
@@ -62,7 +64,21 @@ Renderer::~Renderer()
 
     for (IRenderEngine* const pEngine : _rgpEngines)
     {
-        LOG_IF_FAILED(_PaintFrameForEngine(pEngine));
+        auto tries = maxRetriesForRenderEngine;
+        while (tries > 0)
+        {
+            const auto hr = _PaintFrameForEngine(pEngine);
+            if (E_PENDING == hr)
+            {
+                if (--tries == 0)
+                {
+                    FAIL_FAST_HR_MSG(E_UNEXPECTED, "A rendering engine required too many retries.");
+                }
+                continue;
+            }
+            LOG_IF_FAILED(hr);
+            break;
+        }
     }
 
     return S_OK;
diff --git a/src/renderer/dx/DxRenderer.cpp b/src/renderer/dx/DxRenderer.cpp
index e8cb5589de6..3d52a4b68d1 100644
--- a/src/renderer/dx/DxRenderer.cpp
+++ b/src/renderer/dx/DxRenderer.cpp
@@ -866,15 +866,31 @@ void DxEngine::_InvalidOr(RECT rc) noexcept
 // Arguments:
 // - <none>
 // Return Value:
-// - S_OK or relevant DirectX error
+// - S_OK on success, E_PENDING to indicate a retry or a relevant DirectX error
 [[nodiscard]] HRESULT DxEngine::Present() noexcept
 {
     if (_presentReady)
     {
         try
         {
-            FAIL_FAST_IF_FAILED(_dxgiSwapChain->Present(1, 0));
-            /*FAIL_FAST_IF_FAILED(_dxgiSwapChain->Present1(1, 0, &_presentParams));*/
+            HRESULT hr = S_OK;
+
+            hr = _dxgiSwapChain->Present(1, 0);
+            /*hr = _dxgiSwapChain->Present1(1, 0, &_presentParams);*/
+
+            if (FAILED(hr))
+            {
+                // These two error codes are indicated for destroy-and-recreate
+                if (hr == DXGI_ERROR_DEVICE_REMOVED || hr == DXGI_ERROR_DEVICE_RESET)
+                {
+                    // We don't need to end painting here, as the renderer has done it for us.
+                    _ReleaseDeviceResources();
+                    FAIL_FAST_IF_FAILED(InvalidateAll());
+                    return E_PENDING; // Indicate a retry to the renderer.
+                }
+
+                FAIL_FAST_HR(hr);
+            }
 
             RETURN_IF_FAILED(_CopyFrontToBack());
             _presentReady = false;