AtlasEngine: Fix various ClearType rendering issues (#12278)

This commit fixes the following issues when ClearType rendering is enabled:
* Colored glyphs are now drawn using grayscale AA, similar to all current
  browsers. This ensures proper gamma correctness during blending in our
  shader, while generally making no discernable difference for legibility.
* Our ClearType shader only emits fully opaque colors, just like the official
  ClearType implementation. Due to this we need to force grayscale AA if the
  user specifies both ClearType and a background image, as the image would
  otherwise not be visible.

## PR Checklist
* [x] I work here
* [x] Tests added/passed

## Validation Steps Performed
* Grayscale AA when drawing emojis ✅
* Grayscale AA when using a background image ✅

src/renderer/atlas/AtlasEngine.api.cpp

@@ -311,10 +311,11 @@ HRESULT AtlasEngine::Enable() noexcept
 
 void AtlasEngine::SetAntialiasingMode(const D2D1_TEXT_ANTIALIAS_MODE antialiasingMode) noexcept
 {
-    const auto mode = gsl::narrow_cast<u16>(antialiasingMode);
+    const auto mode = gsl::narrow_cast<u8>(antialiasingMode);
     if (_api.antialiasingMode != mode)
     {
         _api.antialiasingMode = mode;
+        _resolveAntialiasingMode();
         WI_SetFlag(_api.invalidations, ApiInvalidations::Font);
     }
 }
@@ -330,6 +331,7 @@ void AtlasEngine::EnableTransparentBackground(const bool isTransparent) noexcept
     if (_api.backgroundOpaqueMixin != mixin)
     {
         _api.backgroundOpaqueMixin = mixin;
+        _resolveAntialiasingMode();
         WI_SetFlag(_api.invalidations, ApiInvalidations::SwapChain);
     }
 }
@@ -499,6 +501,15 @@ void AtlasEngine::UpdateHyperlinkHoveredId(const uint16_t hoveredId) noexcept
 
 #pragma endregion
 
+void AtlasEngine::_resolveAntialiasingMode() noexcept
+{
+    // If the user asks for ClearType, but also for a transparent background
+    // (which our ClearType shader doesn't simultaneously support)
+    // then we need to sneakily force the renderer to grayscale AA.
+    const auto forceGrayscaleAA = _api.antialiasingMode == D2D1_TEXT_ANTIALIAS_MODE_CLEARTYPE && !_api.backgroundOpaqueMixin;
+    _api.realizedAntialiasingMode = forceGrayscaleAA ? D2D1_TEXT_ANTIALIAS_MODE_GRAYSCALE : _api.antialiasingMode;
+}
+
 void AtlasEngine::_resolveFontMetrics(const FontInfoDesired& fontInfoDesired, FontInfo& fontInfo, FontMetrics* fontMetrics) const
 {
     auto requestedFaceName = fontInfoDesired.GetFaceName().c_str();

src/renderer/atlas/AtlasEngine.h

@@ -614,6 +614,7 @@ namespace Microsoft::Console::Render
         void _emplaceGlyph(IDWriteFontFace* fontFace, size_t bufferPos1, size_t bufferPos2);
 
         // AtlasEngine.api.cpp
+        void _resolveAntialiasingMode() noexcept;
         void _resolveFontMetrics(const FontInfoDesired& fontInfoDesired, FontInfo& fontInfo, FontMetrics* fontMetrics = nullptr) const;
 
         // AtlasEngine.r.cpp
@@ -754,7 +755,8 @@ namespace Microsoft::Console::Render
             wil::unique_handle swapChainHandle;
             HWND hwnd = nullptr;
             u16 dpi = USER_DEFAULT_SCREEN_DPI; // changes are flagged as ApiInvalidations::Font|Size
-            u16 antialiasingMode = D2D1_TEXT_ANTIALIAS_MODE_CLEARTYPE; // changes are flagged as ApiInvalidations::Font
+            u8 antialiasingMode = D2D1_TEXT_ANTIALIAS_MODE_CLEARTYPE; // changes are flagged as ApiInvalidations::Font
+            u8 realizedAntialiasingMode = D2D1_TEXT_ANTIALIAS_MODE_CLEARTYPE; // caches antialiasingMode, depends on antialiasingMode and backgroundOpaqueMixin, see _resolveAntialiasingMode
 
             ApiInvalidations invalidations = ApiInvalidations::Device;
         } _api;

src/renderer/atlas/AtlasEngine.r.cpp

@@ -113,7 +113,7 @@ void AtlasEngine::_setShaderResources() const
 
 void AtlasEngine::_updateConstantBuffer() const noexcept
 {
-    const auto useClearType = _api.antialiasingMode == D2D1_TEXT_ANTIALIAS_MODE_CLEARTYPE;
+    const auto useClearType = _api.realizedAntialiasingMode == D2D1_TEXT_ANTIALIAS_MODE_CLEARTYPE;
 
     ConstBuffer data;
     data.viewport.x = 0;
@@ -273,7 +273,9 @@ void AtlasEngine::_reserveScratchpadSize(u16 minWidth)
         // We don't really use D2D for anything except DWrite, but it
         // can't hurt to ensure that everything it does is pixel aligned.
         _r.d2dRenderTarget->SetAntialiasMode(D2D1_ANTIALIAS_MODE_ALIASED);
-        _r.d2dRenderTarget->SetTextAntialiasMode(static_cast<D2D1_TEXT_ANTIALIAS_MODE>(_api.antialiasingMode));
+        // In case _api.realizedAntialiasingMode is D2D1_TEXT_ANTIALIAS_MODE_CLEARTYPE we'll
+        // continuously adjust it in AtlasEngine::_drawGlyph. See _drawGlyph.
+        _r.d2dRenderTarget->SetTextAntialiasMode(static_cast<D2D1_TEXT_ANTIALIAS_MODE>(_api.realizedAntialiasingMode));
         // Ensure that D2D uses the exact same gamma as our shader uses.
         _r.d2dRenderTarget->SetTextRenderingParams(renderingParams.get());
     }
@@ -311,6 +313,7 @@ void AtlasEngine::_drawGlyph(const AtlasQueueItem& item) const
     const auto charsLength = key->charCount;
     const auto cells = static_cast<u32>(key->attributes.cellCount);
     const auto textFormat = _getTextFormat(key->attributes.bold, key->attributes.italic);
+    const auto coloredGlyph = WI_IsFlagSet(value->flags, CellFlags::ColoredGlyph);
 
     // See D2DFactory::DrawText
     wil::com_ptr<IDWriteTextLayout> textLayout;
@@ -323,7 +326,16 @@ void AtlasEngine::_drawGlyph(const AtlasQueueItem& item) const
     auto options = D2D1_DRAW_TEXT_OPTIONS_NONE;
     // D2D1_DRAW_TEXT_OPTIONS_ENABLE_COLOR_FONT enables a bunch of internal machinery
     // which doesn't have to run if we know we can't use it anyways in the shader.
-    WI_SetFlagIf(options, D2D1_DRAW_TEXT_OPTIONS_ENABLE_COLOR_FONT, WI_IsFlagSet(value->flags, CellFlags::ColoredGlyph));
+    WI_SetFlagIf(options, D2D1_DRAW_TEXT_OPTIONS_ENABLE_COLOR_FONT, coloredGlyph);
+
+    // Colored glyphs cannot be drawn in linear gamma.
+    // That's why we're simply alpha-blending them in the shader.
+    // In order for this to work correctly we have to prevent them from being drawn
+    // with ClearType, because we would then lack the alpha channel for the glyphs.
+    if (_api.realizedAntialiasingMode == D2D1_TEXT_ANTIALIAS_MODE_CLEARTYPE)
+    {
+        _r.d2dRenderTarget->SetTextAntialiasMode(coloredGlyph ? D2D1_TEXT_ANTIALIAS_MODE_GRAYSCALE : D2D1_TEXT_ANTIALIAS_MODE_CLEARTYPE);
+    }
 
     _r.d2dRenderTarget->BeginDraw();
     // We could call