Add support for renderer backoff, don't FAIL_FAST on 3x failures, add UI

src/cascadia/TerminalControl/Resources/en-US/Resources.resw

@@ -164,4 +164,10 @@
   <data name="SearchBox_Close.[using:Windows.UI.Xaml.Automation]AutomationProperties.Name" xml:space="preserve">
     <value>Close Search Box</value>
   </data>
-</root>
+  <data name="TermControl_RendererFailedTextBlock.Text" xml:space="preserve">
+    <value>This terminal has encountered an issue with the graphics driver and it could not recover in time. It has been suspended.</value>
+  </data>
+  <data name="TermControl_RendererRetryButton.Content" xml:space="preserve">
+    <value>Resume</value>
+  </data>
+</root>

src/cascadia/TerminalControl/TermControl.cpp

@@ -534,6 +534,13 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
             _renderer = std::make_unique<::Microsoft::Console::Render::Renderer>(_terminal.get(), nullptr, 0, std::move(renderThread));
             ::Microsoft::Console::Render::IRenderTarget& renderTarget = *_renderer;
 
+            _renderer->SetRendererEnteredErrorStateCallback([weakThis = get_weak()]() {
+                if (auto strongThis{ weakThis.get() })
+                {
+                    strongThis->_RendererEnteredErrorState();
+                }
+            });
+
             THROW_IF_FAILED(localPointerToThread->Initialize(_renderer.get()));
 
             // Set up the DX Engine
@@ -2511,6 +2518,31 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
         e.DragUIOverride().IsGlyphVisible(false);
     }
 
+    // Method Description:
+    // - Produces the error dialog that notifies the user that rendering cannot proceed.
+    winrt::fire_and_forget TermControl::_RendererEnteredErrorState()
+    {
+        auto strongThis{ get_strong() };
+        co_await Dispatcher(); // pop up onto the UI thread
+
+        if (auto loadedUiElement{ FindName(L"RendererFailedNotice") })
+        {
+            if (auto uiElement{ loadedUiElement.try_as<::winrt::Windows::UI::Xaml::UIElement>() })
+            {
+                uiElement.Visibility(Visibility::Visible);
+            }
+        }
+    }
+
+    // Method Description:
+    // - Responds to the Click event on the button that will re-enable the renderer.
+    void TermControl::_RenderRetryButton_Click(IInspectable const& /*sender*/, IInspectable const& /*args*/)
+    {
+        // It's already loaded if we get here, so just hide it.
+        RendererFailedNotice().Visibility(Visibility::Collapsed);
+        _renderer->ResetErrorStateAndResume();
+    }
+
     // -------------------------------- WinRT Events ---------------------------------
     // Winrt events need a method for adding a callback to the event and removing the callback.
     // These macros will define them both for you.

src/cascadia/TerminalControl/TermControl.h

@@ -79,6 +79,8 @@ namespace winrt::Microsoft::Terminal::TerminalControl::implementation
 
         winrt::fire_and_forget RenderEngineSwapChainChanged();
         void _AttachDxgiSwapChainToXaml(IDXGISwapChain1* swapChain);
+        winrt::fire_and_forget _RendererEnteredErrorState();
+        void _RenderRetryButton_Click(IInspectable const& button, IInspectable const& args);
 
         void CreateSearchBoxControl();
 

src/cascadia/TerminalControl/TermControl.xaml

@@ -77,6 +77,23 @@
                                CurrentCursorPosition="_CurrentCursorPositionHandler"
                                CurrentFontInfo="_FontInfoHandler" />
 
+        <Grid x:Name="RendererFailedNotice"
+              x:Load="False"
+              HorizontalAlignment="Center"
+              VerticalAlignment="Center">
+            <Border Background="{ThemeResource SystemControlBackgroundAltHighBrush}"
+                    BorderBrush="{ThemeResource SystemAccentColor}"
+                    Margin="8,8,8,8"
+                    Padding="8,8,8,8"
+                    BorderThickness="2,2,2,2"
+                    CornerRadius="{ThemeResource OverlayCornerRadius}">
+                <StackPanel>
+                    <TextBlock HorizontalAlignment="Center" x:Uid="TermControl_RendererFailedTextBlock" TextWrapping="WrapWholeWords"/>
+                    <Button Click="_RenderRetryButton_Click" x:Uid="TermControl_RendererRetryButton" HorizontalAlignment="Right" />
+                </StackPanel>
+            </Border>
+        </Grid>
+
     </Grid>
 
 </UserControl>

src/renderer/base/renderer.cpp

@@ -11,6 +11,8 @@ using namespace Microsoft::Console::Render;
 using namespace Microsoft::Console::Types;
 
 static constexpr auto maxRetriesForRenderEngine = 3;
+// The renderer will wait this number of milliseconds * how many tries have elapsed before trying again.
+static constexpr auto renderBackoffBaseTimeMilliseconds{ 150 };
 
 // Routine Description:
 // - Creates a new renderer controller for a console.
@@ -78,8 +80,20 @@ Renderer::~Renderer()
             {
                 if (--tries == 0)
                 {
-                    FAIL_FAST_HR_MSG(E_UNEXPECTED, "A rendering engine required too many retries.");
+                    // Stop trying.
+                    _pThread->DisablePainting();
+                    if (_pfnRendererEnteredErrorState)
+                    {
+                        _pfnRendererEnteredErrorState();
+                    }
+                    // If there's no callback, we still don't want to FAIL_FAST: the renderer going black
+                    // isn't near as bad as the entire application aborting. We're a component. We shouldn't
+                    // abort applications that host us.
+                    return S_FALSE;
                 }
+                // Add a bit of backoff.
+                // Sleep 150ms, 300ms, 450ms before failing out and disabling the renderer.
+                Sleep(renderBackoffBaseTimeMilliseconds * (maxRetriesForRenderEngine - tries));
                 continue;
             }
             LOG_IF_FAILED(hr);
@@ -1077,3 +1091,23 @@ void Renderer::AddRenderEngine(_In_ IRenderEngine* const pEngine)
     THROW_HR_IF_NULL(E_INVALIDARG, pEngine);
     _rgpEngines.push_back(pEngine);
 }
+
+// Method Description:
+// - Registers a callback that will be called when this renderer gives up.
+//   An application consuming a renderer can use this to display auxiliary Retry UI
+// Arguments:
+// - pfn: the callback
+// Return Value:
+// - <none>
+void Renderer::SetRendererEnteredErrorStateCallback(std::function<void()> pfn)
+{
+    _pfnRendererEnteredErrorState = std::move(pfn);
+}
+
+// Method Description:
+// - Attempts to restart the renderer.
+void Renderer::ResetErrorStateAndResume()
+{
+    // because we're not stateful (we could be in the future), all we want to do is reenable painting.
+    EnablePainting();
+}

src/renderer/base/renderer.hpp

@@ -76,6 +76,9 @@ namespace Microsoft::Console::Render
 
         void AddRenderEngine(_In_ IRenderEngine* const pEngine) override;
 
+        void SetRendererEnteredErrorStateCallback(std::function<void()> pfn);
+        void ResetErrorStateAndResume();
+
     private:
         std::deque<IRenderEngine*> _rgpEngines;
 
@@ -128,6 +131,8 @@ namespace Microsoft::Console::Render
         // These are only actually effective/on in Debug builds when the flag is set using an attached debugger.
         bool _fDebug = false;
 
+        std::function<void()> _pfnRendererEnteredErrorState;
+
 #ifdef UNIT_TESTING
         friend class ConptyOutputTests;
 #endif

src/renderer/base/thread.cpp

@@ -26,6 +26,7 @@ RenderThread::~RenderThread()
     if (_hThread)
     {
         _fKeepRunning = false; // stop loop after final run
+        EnablePainting(); // if we want to get the last frame out, we need to make sure it's enabled
         SignalObjectAndWait(_hEvent, _hThread, INFINITE, FALSE); // signal final paint and wait for thread to finish.
 
         CloseHandle(_hThread);
@@ -231,6 +232,11 @@ void RenderThread::EnablePainting()
     SetEvent(_hPaintEnabledEvent);
 }
 
+void RenderThread::DisablePainting()
+{
+    ResetEvent(_hPaintEnabledEvent);
+}
+
 void RenderThread::WaitForPaintCompletionAndDisable(const DWORD dwTimeoutMs)
 {
     // When rendering takes place via DirectX, and a console application

src/renderer/base/thread.hpp

@@ -30,6 +30,7 @@ namespace Microsoft::Console::Render
         void NotifyPaint() override;
 
         void EnablePainting() override;
+        void DisablePainting() override;
         void WaitForPaintCompletionAndDisable(const DWORD dwTimeoutMs) override;
 
     private:

src/renderer/inc/IRenderThread.hpp

@@ -26,6 +26,7 @@ namespace Microsoft::Console::Render
 
         virtual void NotifyPaint() = 0;
         virtual void EnablePainting() = 0;
+        virtual void DisablePainting() = 0;
         virtual void WaitForPaintCompletionAndDisable(const DWORD dwTimeoutMs) = 0;
 
     protected: