Grab bag of UTF8 string support in IDE features

src/EditorFeatures/CSharp/BraceMatching/StringLiteralBraceMatcher.cs

@@ -34,18 +34,11 @@ public StringLiteralBraceMatcher()
             {
                 if (token.IsKind(SyntaxKind.StringLiteralToken))
                 {
-                    if (token.IsVerbatimStringLiteral())
-                    {
-                        return new BraceMatchingResult(
-                            new TextSpan(token.SpanStart, 2),
-                            new TextSpan(token.Span.End - 1, 1));
-                    }
-                    else
-                    {
-                        return new BraceMatchingResult(
-                            new TextSpan(token.SpanStart, 1),
-                            new TextSpan(token.Span.End - 1, 1));
-                    }
+                    return GetSimpleStringBraceMatchingResult(token, endTokenLength: 1);
+                }
+                else if (token.IsKind(SyntaxKind.UTF8StringLiteralToken))
+                {
+                    return GetSimpleStringBraceMatchingResult(token, endTokenLength: 3);
                 }
                 else if (token.IsKind(SyntaxKind.InterpolatedStringStartToken, SyntaxKind.InterpolatedVerbatimStringStartToken))
                 {
@@ -65,5 +58,21 @@ public StringLiteralBraceMatcher()
 
             return null;
         }
+
+        private static BraceMatchingResult GetSimpleStringBraceMatchingResult(SyntaxToken token, int endTokenLength)
+        {
+            if (token.IsVerbatimStringLiteral())
+            {
+                return new BraceMatchingResult(
+                    new TextSpan(token.SpanStart, 2),
+                    new TextSpan(token.Span.End - endTokenLength, endTokenLength));
+            }
+            else
+            {
+                return new BraceMatchingResult(
+                    new TextSpan(token.SpanStart, 1),
+                    new TextSpan(token.Span.End - endTokenLength, endTokenLength));
+            }
+        }
     }
 }

src/EditorFeatures/CSharp/TextStructureNavigation/CSharpTextStructureNavigatorProvider.cs

@@ -11,6 +11,7 @@
 using Microsoft.VisualStudio.Text;
 using Microsoft.VisualStudio.Text.Operations;
 using Microsoft.VisualStudio.Utilities;
+using Roslyn.Utilities;
 
 namespace Microsoft.CodeAnalysis.Editor.CSharp.TextStructureNavigation
 {
@@ -36,12 +37,15 @@ protected override bool IsWithinNaturalLanguage(SyntaxToken token, int position)
             switch (token.Kind())
             {
                 case SyntaxKind.StringLiteralToken:
+                case SyntaxKind.UTF8StringLiteralToken:
                     // This, in combination with the override of GetExtentOfWordFromToken() below, treats the closing
                     // quote as a separate token.  This maintains behavior with VS2013.
                     return !IsAtClosingQuote(token, position);
 
                 case SyntaxKind.SingleLineRawStringLiteralToken:
                 case SyntaxKind.MultiLineRawStringLiteralToken:
+                case SyntaxKind.UTF8SingleLineRawStringLiteralToken:
+                case SyntaxKind.UTF8MultiLineRawStringLiteralToken:
                     {
                         // Like with normal string literals, treat the closing quotes as as the end of the string so that
                         // navigation ends there and doesn't go past them.
@@ -66,6 +70,13 @@ private static int GetStartOfRawStringLiteralEndDelimiter(SyntaxToken token)
             var text = token.ToString();
             var start = 0;
             var end = text.Length;
+
+            if (token.IsKind(SyntaxKind.UTF8MultiLineRawStringLiteralToken, SyntaxKind.UTF8SingleLineRawStringLiteralToken))
+            {
+                // Skip past the u8 suffix
+                end -= "u8".Length;
+            }
+
             while (start < end && text[start] == '"')
                 start++;
 
@@ -76,19 +87,27 @@ private static int GetStartOfRawStringLiteralEndDelimiter(SyntaxToken token)
         }
 
         private static bool IsAtClosingQuote(SyntaxToken token, int position)
-            => position == token.Span.End - 1 && token.Text[^1] == '"';
+            => token.Kind() switch
+            {
+                SyntaxKind.StringLiteralToken => position == token.Span.End - 1 && token.Text[^1] == '"',
+                SyntaxKind.UTF8StringLiteralToken => position == token.Span.End - 3 && token.Text is [.., '"', 'u' or 'U', '8'],
+                _ => throw ExceptionUtilities.Unreachable
+            };
 
         protected override TextExtent GetExtentOfWordFromToken(SyntaxToken token, SnapshotPoint position)
         {
-            if (token.Kind() == SyntaxKind.StringLiteralToken && IsAtClosingQuote(token, position.Position))
+            if (token.IsKind(SyntaxKind.StringLiteralToken, SyntaxKind.UTF8StringLiteralToken) && IsAtClosingQuote(token, position.Position))
             {
                 // Special case to treat the closing quote of a string literal as a separate token.  This allows the
                 // cursor to stop during word navigation (Ctrl+LeftArrow, etc.) immediately before AND after the
                 // closing quote, just like it did in VS2013 and like it currently does for interpolated strings.
-                var span = new Span(position.Position, 1);
+                var span = new Span(position.Position, token.Span.End - position.Position);
                 return new TextExtent(new SnapshotSpan(position.Snapshot, span), isSignificant: true);
             }
-            else if (token.Kind() is SyntaxKind.SingleLineRawStringLiteralToken or SyntaxKind.MultiLineRawStringLiteralToken)
+            else if (token.IsKind(SyntaxKind.SingleLineRawStringLiteralToken,
+                SyntaxKind.MultiLineRawStringLiteralToken,
+                SyntaxKind.UTF8SingleLineRawStringLiteralToken,
+                SyntaxKind.UTF8MultiLineRawStringLiteralToken))
             {
                 var delimiterStart = GetStartOfRawStringLiteralEndDelimiter(token);
                 return new TextExtent(new SnapshotSpan(position.Snapshot, Span.FromBounds(delimiterStart, token.Span.End)), isSignificant: true);

src/EditorFeatures/CSharpTest/BraceMatching/CSharpBraceMatcherTests.cs

@@ -505,6 +505,123 @@ public async Task TestInterpolatedString13()
             await TestAsync(code, expected);
         }
 
+        [Fact, Trait(Traits.Feature, Traits.Features.BraceMatching)]
+        public async Task TestUTF8String1()
+        {
+            var code = @"public class C { string s = $$""Goo""u8; }";
+            var expected = @"public class C { string s = ""Goo[|""u8|]; }";
+
+            await TestAsync(code, expected);
+        }
+
+        [Fact, Trait(Traits.Feature, Traits.Features.BraceMatching)]
+        public async Task TestUTF8String2()
+        {
+            var code = @"public class C { string s = ""$$Goo""u8; }";
+            var expected = @"public class C { string s = ""Goo[|""u8|]; }";
+
+            await TestAsync(code, expected);
+        }
+
+        [Fact, Trait(Traits.Feature, Traits.Features.BraceMatching)]
+        public async Task TestUTF8String3()
+        {
+            var code = @"public class C { string s = ""Goo$$""u8; }";
+            var expected = @"public class C { string s = [|""|]Goo""u8; }";
+
+            await TestAsync(code, expected);
+        }
+
+        [Fact, Trait(Traits.Feature, Traits.Features.BraceMatching)]
+        public async Task TestUTF8String4()
+        {
+            var code = @"public class C { string s = ""Goo""$$u8; }";
+            var expected = @"public class C { string s = [|""|]Goo""u8; }";
+
+            await TestAsync(code, expected);
+        }
+
+        [Fact, Trait(Traits.Feature, Traits.Features.BraceMatching)]
+        public async Task TestUTF8String5()
+        {
+            var code = @"public class C { string s = ""Goo""u$$8; }";
+            var expected = @"public class C { string s = [|""|]Goo""u8; }";
+
+            await TestAsync(code, expected);
+        }
+
+        [Fact, Trait(Traits.Feature, Traits.Features.BraceMatching)]
+        public async Task TestUTF8String6()
+        {
+            var code = @"public class C { string s = ""Goo""u8$$; }";
+            var expected = @"public class C { string s = [|""|]Goo""u8; }";
+
+            await TestAsync(code, expected);
+        }
+
+        [Fact, Trait(Traits.Feature, Traits.Features.BraceMatching)]
+        public async Task TestVerbatimUTF8String1()
+        {
+            var code = @"public class C { string s = $$@""Goo""u8; }";
+            var expected = @"public class C { string s = @""Goo[|""u8|]; }";
+
+            await TestAsync(code, expected);
+        }
+
+        [Fact, Trait(Traits.Feature, Traits.Features.BraceMatching)]
+        public async Task TestVerbatimUTF8String2()
+        {
+            var code = @"public class C { string s = @$$""Goo""u8; }";
+            var expected = @"public class C { string s = @""Goo[|""u8|]; }";
+
+            await TestAsync(code, expected);
+        }
+
+        [Fact, Trait(Traits.Feature, Traits.Features.BraceMatching)]
+        public async Task TestVerbatimUTF8String3()
+        {
+            var code = @"public class C { string s = @""$$Goo""u8; }";
+            var expected = @"public class C { string s = @""Goo[|""u8|]; }";
+
+            await TestAsync(code, expected);
+        }
+
+        [Fact, Trait(Traits.Feature, Traits.Features.BraceMatching)]
+        public async Task TestVerbatimUTF8String4()
+        {
+            var code = @"public class C { string s = @""Goo$$""u8; }";
+            var expected = @"public class C { string s = [|@""|]Goo""u8; }";
+
+            await TestAsync(code, expected);
+        }
+
+        [Fact, Trait(Traits.Feature, Traits.Features.BraceMatching)]
+        public async Task TestVerbatimUTF8String5()
+        {
+            var code = @"public class C { string s = @""Goo""$$u8; }";
+            var expected = @"public class C { string s = [|@""|]Goo""u8; }";
+
+            await TestAsync(code, expected);
+        }
+
+        [Fact, Trait(Traits.Feature, Traits.Features.BraceMatching)]
+        public async Task TestVerbatimUTF8String6()
+        {
+            var code = @"public class C { string s = @""Goo""u$$8; }";
+            var expected = @"public class C { string s = [|@""|]Goo""u8; }";
+
+            await TestAsync(code, expected);
+        }
+
+        [Fact, Trait(Traits.Feature, Traits.Features.BraceMatching)]
+        public async Task TestVerbatimUTF8String7()
+        {
+            var code = @"public class C { string s = @""Goo""u8$$; }";
+            var expected = @"public class C { string s = [|@""|]Goo""u8; }";
+
+            await TestAsync(code, expected);
+        }
+
         [WorkItem(7120, "https://github.com/dotnet/roslyn/issues/7120")]
         [WpfFact, Trait(Traits.Feature, Traits.Features.BraceMatching)]
         public async Task TestConditionalDirectiveWithSingleMatchingDirective()

src/EditorFeatures/CSharpTest/ConvertNamespace/ConvertToFileScopedNamespaceAnalyzerTests.cs

@@ -677,6 +677,102 @@ void M()
             }.RunAsync();
         }
 
+        [Fact]
+        public async Task TestConvertToFileScopedWithMultiLineRawString()
+        {
+            await new VerifyCS.Test
+            {
+                TestCode = @"
+[|namespace N|]
+{
+    class C
+    {
+        void M()
+        {
+            System.Console.WriteLine(""""""
+    a
+        b
+            c
+                d
+                    e
+    """""");
+        }
+    }
+}
+",
+                FixedCode = @"
+namespace $$N;
+
+class C
+{
+    void M()
+    {
+        System.Console.WriteLine(""""""
+    a
+        b
+            c
+                d
+                    e
+    """""");
+    }
+}
+",
+                LanguageVersion = LanguageVersion.Preview,
+                Options =
+                {
+                    { CSharpCodeStyleOptions.NamespaceDeclarations, NamespaceDeclarationPreference.FileScoped }
+                }
+            }.RunAsync();
+        }
+
+        [Fact]
+        public async Task TestConvertToFileScopedWithUTF8MultiLineRawString()
+        {
+            await new VerifyCS.Test
+            {
+                TestCode = @"
+[|namespace N|]
+{
+    class C
+    {
+        void M()
+        {
+            System.Console.WriteLine(""""""
+    a
+        b
+            c
+                d
+                    e
+    """"""u8);
+        }
+    }
+}
+",
+                FixedCode = @"
+namespace $$N;
+
+class C
+{
+    void M()
+    {
+        System.Console.WriteLine(""""""
+    a
+        b
+            c
+                d
+                    e
+    """"""u8);
+    }
+}
+",
+                LanguageVersion = LanguageVersion.Preview,
+                Options =
+                {
+                    { CSharpCodeStyleOptions.NamespaceDeclarations, NamespaceDeclarationPreference.FileScoped }
+                }
+            }.RunAsync();
+        }
+
         [Fact]
         public async Task TestConvertToFileScopedSingleLineNamespace1()
         {