Use the roslyn tokenizer (#10702)

This is the big one: using the Roslyn tokenizer during Razor parsing.
I've done my best to separate out various pieces into separate commits
to make the review a bit simpler, but there's no getting around the
lexer change being complicated. I would recommend commit-by-commit to
make it as simple as possible. Fixes
#10568, fixes
#7084.

src/Compiler/Microsoft.AspNetCore.Razor.Language/test/DefaultRazorIntermediateNodeLoweringPhaseIntegrationTest.cs

@@ -525,6 +525,7 @@ public DesignTimeOptionsFeature(bool designTime)
         public void Configure(RazorParserOptionsBuilder options)
         {
             options.SetDesignTime(_designTime);
+            options.UseRoslynTokenizer = true;
         }
 
         public void Configure(RazorCodeGenerationOptionsBuilder options)

src/Compiler/Microsoft.AspNetCore.Razor.Language/test/DefaultRazorIntermediateNodeLoweringPhaseTest.cs

@@ -30,7 +30,11 @@ public void Execute_AutomaticallyImportsSingleLineSinglyOccurringDirective()
             b.Features.Add(new DefaultRazorCodeGenerationOptionsFeature(designTime: false));
             b.AddDirective(directive);
         });
-        var options = RazorParserOptions.Create(builder => builder.Directives.Add(directive));
+        var options = RazorParserOptions.Create(builder =>
+        {
+            builder.Directives.Add(directive);
+            builder.UseRoslynTokenizer = true;
+        });
         var importSource = TestRazorSourceDocument.Create("@custom \"hello\"", filePath: "import.cshtml");
         var codeDocument = TestRazorCodeDocument.Create("<p>NonDirective</p>");
         codeDocument.SetSyntaxTree(RazorSyntaxTree.Parse(codeDocument.Source, options));
@@ -65,7 +69,11 @@ public void Execute_AutomaticallyOverridesImportedSingleLineSinglyOccurringDirec
             b.Features.Add(new DefaultRazorCodeGenerationOptionsFeature(designTime: false));
             b.AddDirective(directive);
         });
-        var options = RazorParserOptions.Create(builder => builder.Directives.Add(directive));
+        var options = RazorParserOptions.Create(builder =>
+        {
+            builder.Directives.Add(directive);
+            builder.UseRoslynTokenizer = true;
+        });
         var importSource = TestRazorSourceDocument.Create("@custom \"hello\"", filePath: "import.cshtml");
         var codeDocument = TestRazorCodeDocument.Create("@custom \"world\"");
         codeDocument.SetSyntaxTree(RazorSyntaxTree.Parse(codeDocument.Source, options));
@@ -100,7 +108,11 @@ public void Execute_AutomaticallyOverridesImportedSingleLineSinglyOccurringDirec
             b.Features.Add(new DefaultRazorCodeGenerationOptionsFeature(designTime: false));
             b.AddDirective(directive);
         });
-        var options = RazorParserOptions.Create(builder => builder.Directives.Add(directive));
+        var options = RazorParserOptions.Create(builder =>
+        {
+            builder.Directives.Add(directive);
+            builder.UseRoslynTokenizer = true;
+        });
         var importSource1 = TestRazorSourceDocument.Create("@custom \"hello\"", filePath: "import1.cshtml");
         var importSource2 = TestRazorSourceDocument.Create("@custom \"world\"", filePath: "import2.cshtml");
         var codeDocument = TestRazorCodeDocument.Create("<p>NonDirective</p>");
@@ -136,6 +148,7 @@ public void Execute_DoesNotImportNonFileScopedSinglyOccurringDirectives_Block()
         {
             builder.Directives.Add(codeBlockDirective);
             builder.Directives.Add(razorBlockDirective);
+            builder.UseRoslynTokenizer = true;
         });
         var importSource = TestRazorSourceDocument.Create(
 @"@code ""code block"" { }
@@ -166,7 +179,11 @@ public void Execute_ErrorsForCodeBlockFileScopedSinglyOccurringDirectives()
             b.Features.Add(new DefaultRazorCodeGenerationOptionsFeature(designTime: false));
             b.AddDirective(directive);
         });
-        var options = RazorParserOptions.Create(builder => builder.Directives.Add(directive));
+        var options = RazorParserOptions.Create(builder =>
+        {
+            builder.Directives.Add(directive);
+            builder.UseRoslynTokenizer = true;
+        });
         var importSource = TestRazorSourceDocument.Create("@custom { }", filePath: "import.cshtml");
         var codeDocument = TestRazorCodeDocument.Create("<p>NonDirective</p>");
         codeDocument.SetSyntaxTree(RazorSyntaxTree.Parse(codeDocument.Source, options));
@@ -196,7 +213,11 @@ public void Execute_ErrorsForRazorBlockFileScopedSinglyOccurringDirectives()
             b.Features.Add(new DefaultRazorCodeGenerationOptionsFeature(designTime: false));
             b.AddDirective(directive);
         });
-        var options = RazorParserOptions.Create(builder => builder.Directives.Add(directive));
+        var options = RazorParserOptions.Create(builder =>
+        {
+            builder.Directives.Add(directive);
+            builder.UseRoslynTokenizer = true;
+        });
         var importSource = TestRazorSourceDocument.Create("@custom { }", filePath: "import.cshtml");
         var codeDocument = TestRazorCodeDocument.Create("<p>NonDirective</p>");
         codeDocument.SetSyntaxTree(RazorSyntaxTree.Parse(codeDocument.Source, options));
@@ -247,8 +268,13 @@ public void Execute_CollatesSyntaxDiagnosticsFromSourceDocument()
             b.Phases.Add(phase);
             b.Features.Add(new DefaultRazorCodeGenerationOptionsFeature(designTime: false));
         });
+
+        var options = RazorParserOptions.Create(builder =>
+        {
+            builder.UseRoslynTokenizer = true;
+        });
         var codeDocument = TestRazorCodeDocument.Create("<p class=@(");
-        codeDocument.SetSyntaxTree(RazorSyntaxTree.Parse(codeDocument.Source));
+        codeDocument.SetSyntaxTree(RazorSyntaxTree.Parse(codeDocument.Source, options));
 
         // Act
         phase.Execute(codeDocument);
@@ -271,12 +297,16 @@ public void Execute_CollatesSyntaxDiagnosticsFromImportDocuments()
             b.Features.Add(new DefaultRazorCodeGenerationOptionsFeature(designTime: false));
         });
 
+        var parseOptions = RazorParserOptions.Create(builder =>
+        {
+            builder.UseRoslynTokenizer = true;
+        });
         var codeDocument = TestRazorCodeDocument.CreateEmpty();
-        codeDocument.SetSyntaxTree(RazorSyntaxTree.Parse(codeDocument.Source));
+        codeDocument.SetSyntaxTree(RazorSyntaxTree.Parse(codeDocument.Source, parseOptions));
         codeDocument.SetImportSyntaxTrees(new[]
         {
-            RazorSyntaxTree.Parse(TestRazorSourceDocument.Create("@ ")),
-            RazorSyntaxTree.Parse(TestRazorSourceDocument.Create("<p @(")),
+            RazorSyntaxTree.Parse(TestRazorSourceDocument.Create("@ "), parseOptions),
+            RazorSyntaxTree.Parse(TestRazorSourceDocument.Create("<p @("), parseOptions),
         }.ToImmutableArray());
         var options = RazorCodeGenerationOptions.CreateDefault();
 

src/Compiler/Microsoft.AspNetCore.Razor.Language/test/Legacy/CSharpTokenizerCommentTest.cs

@@ -55,42 +55,208 @@ public void Next_Returns_Comment_Token_For_Entire_Single_Line_Comment()
     [Fact]
     public void Single_Line_Comment_Is_Terminated_By_Newline()
     {
-        TestTokenizer("// Foo Bar Baz\na", SyntaxFactory.Token(SyntaxKind.CSharpComment, "// Foo Bar Baz"), IgnoreRemaining);
+        TestTokenizer("""
+            // Foo Bar Baz
+            a
+            """, SyntaxFactory.Token(SyntaxKind.CSharpComment, "// Foo Bar Baz"), IgnoreRemaining);
     }
 
     [Fact]
     public void Multi_Line_Comment_In_Single_Line_Comment_Has_No_Effect()
     {
-        TestTokenizer("// Foo/*Bar*/ Baz\na", SyntaxFactory.Token(SyntaxKind.CSharpComment, "// Foo/*Bar*/ Baz"), IgnoreRemaining);
+        TestTokenizer("""
+            // Foo/*Bar*/ Baz
+            a
+            """, SyntaxFactory.Token(SyntaxKind.CSharpComment, "// Foo/*Bar*/ Baz"), IgnoreRemaining);
     }
 
     [Fact]
     public void Next_Returns_Comment_Token_For_Entire_Multi_Line_Comment()
     {
-        TestTokenizer("/* Foo\nBar\nBaz */", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo\nBar\nBaz */"));
+        TestTokenizer("""
+            /* Foo
+            Bar
+            Baz */
+            """, SyntaxFactory.Token(SyntaxKind.CSharpComment, """
+            /* Foo
+            Bar
+            Baz */
+            """));
     }
 
     [Fact]
     public void Multi_Line_Comment_Is_Terminated_By_End_Sequence()
     {
-        TestTokenizer("/* Foo\nBar\nBaz */a", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo\nBar\nBaz */"), IgnoreRemaining);
+        TestTokenizer("""
+            /* Foo
+            Bar
+            Baz */a
+            """, SyntaxFactory.Token(SyntaxKind.CSharpComment, """
+            /* Foo
+            Bar
+            Baz */
+            """), IgnoreRemaining);
     }
 
     [Fact]
     public void Unterminated_Multi_Line_Comment_Captures_To_EOF()
     {
-        TestTokenizer("/* Foo\nBar\nBaz", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo\nBar\nBaz"), IgnoreRemaining);
+        TestTokenizer("""
+            /* Foo
+            Bar
+            Baz
+            """, SyntaxFactory.Token(SyntaxKind.CSharpComment, """
+            /* Foo
+            Bar
+            Baz
+            """), IgnoreRemaining);
     }
 
     [Fact]
     public void Nested_Multi_Line_Comments_Terminated_At_First_End_Sequence()
     {
-        TestTokenizer("/* Foo/*\nBar\nBaz*/ */", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo/*\nBar\nBaz*/"), IgnoreRemaining);
+        TestTokenizer("""
+            /* Foo/*
+            Bar
+            Baz*/ */
+            """, SyntaxFactory.Token(SyntaxKind.CSharpComment, """
+            /* Foo/*
+            Bar
+            Baz*/
+            """), IgnoreRemaining);
     }
 
     [Fact]
     public void Nested_Multi_Line_Comments_Terminated_At_Full_End_Sequence()
     {
-        TestTokenizer("/* Foo\nBar\nBaz* */", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/* Foo\nBar\nBaz* */"), IgnoreRemaining);
+        TestTokenizer("""
+            /* Foo
+            Bar
+            Baz* */
+            """, SyntaxFactory.Token(SyntaxKind.CSharpComment, """
+            /* Foo
+            Bar
+            Baz* */
+            """), IgnoreRemaining);
+    }
+
+    [Fact]
+    public void Next_Returns_CSharpComment_Token_For_Single_Line_Documentation_Comment()
+    {
+        TestTokenizer("/// This is a single line documentation comment", SyntaxFactory.Token(SyntaxKind.CSharpComment, "/// This is a single line documentation comment"));
+    }
+
+    [Fact]
+    public void Single_Line_Documentation_Comment_Is_Terminated_By_And_Owns_Newline()
+    {
+        TestTokenizer("""
+            /// This is a single line documentation comment
+            a
+            """, SyntaxFactory.Token(SyntaxKind.CSharpComment, """
+            /// This is a single line documentation comment
+
+            """), IgnoreRemaining);
+    }
+
+    [Fact]
+    public void Single_Line_Documentation_Comment_With_Multiple_Lines()
+    {
+        TestTokenizer("""
+            /// This is a single line documentation comment
+            /// with multiple lines
+            /// in the comment
+
+            """, SyntaxFactory.Token(SyntaxKind.CSharpComment, """
+            /// This is a single line documentation comment
+            /// with multiple lines
+            /// in the comment
+
+            """), IgnoreRemaining);
+    }
+
+    [Fact]
+    public void Next_Returns_CSharpComment_Token_For_Multi_Line_Documentation_Comment()
+    {
+        TestTokenizer("""
+            /**
+             * This is a
+             * multi-line
+             * documentation comment
+             */
+            """, SyntaxFactory.Token(SyntaxKind.CSharpComment, """
+            /**
+             * This is a
+             * multi-line
+             * documentation comment
+             */
+            """));
+    }
+
+    [Fact]
+    public void Multi_Line_Documentation_Comment_Is_Terminated_By_End_Sequence()
+    {
+        TestTokenizer("""
+            /**
+             * This is a
+             * multi-line
+             * documentation comment
+             */a
+            """, SyntaxFactory.Token(SyntaxKind.CSharpComment, """
+            /**
+             * This is a
+             * multi-line
+             * documentation comment
+             */
+            """), IgnoreRemaining);
+    }
+
+    [Fact]
+    public void Unterminated_Multi_Line_Documentation_Comment_Captures_To_EOF()
+    {
+        TestTokenizer("""
+            /**
+             * This is a
+             * multi-line
+             * documentation comment
+            """, SyntaxFactory.Token(SyntaxKind.CSharpComment, """
+            /**
+             * This is a
+             * multi-line
+             * documentation comment
+            """), IgnoreRemaining);
+    }
+
+    [Fact]
+    public void Nested_Multi_Line_Documentation_Comments_Terminated_At_First_End_Sequence()
+    {
+        TestTokenizer("""
+            /**
+             * This is a
+             * /*nested*/
+             * documentation comment
+             */
+            """, SyntaxFactory.Token(SyntaxKind.CSharpComment, """
+            /**
+             * This is a
+             * /*nested*/
+            """), IgnoreRemaining);
+    }
+
+    [Fact]
+    public void Nested_Multi_Line_Documentation_Comments_Terminated_At_Full_End_Sequence()
+    {
+        TestTokenizer("""
+            /**
+             * This is a
+             * multi-line
+             * documentation comment*
+             */
+            """, SyntaxFactory.Token(SyntaxKind.CSharpComment, """
+            /**
+             * This is a
+             * multi-line
+             * documentation comment*
+             */
+            """), IgnoreRemaining);
     }
 }