Plumb CSharpParseOptions to the tokenizer (#10733)

Does as the tin says. Prerequisite to handling directives, as we'll need the parse options to know what preprocessor symbols are enabled.

src/Compiler/Microsoft.AspNetCore.Razor.Language/test/Legacy/CSharpTokenizerTestBase.cs

@@ -5,9 +5,11 @@
 
 using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
 
+using CSharpParseOptions = Microsoft.CodeAnalysis.CSharp.CSharpParseOptions;
+
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
-public abstract class CSharpTokenizerTestBase : TokenizerTestBase
+public abstract class CSharpTokenizerTestBase : TokenizerTestBase<CSharpParseOptions>
 {
     private static readonly SyntaxToken _ignoreRemaining = SyntaxFactory.Token(SyntaxKind.Marker, string.Empty);
 
@@ -16,11 +18,13 @@ internal override object IgnoreRemaining
         get { return _ignoreRemaining; }
     }
 
-    internal override object CreateTokenizer(SeekableTextReader source)
+    internal override object CreateTokenizer(SeekableTextReader source, CSharpParseOptions parseOptions)
     {
-        return new RoslynCSharpTokenizer(source);
+        return new RoslynCSharpTokenizer(source, parseOptions);
     }
 
+    internal override CSharpParseOptions DefaultTokenizerArg => CSharpParseOptions.Default;
+
     internal void TestSingleToken(string text, SyntaxKind expectedTokenKind)
     {
         TestTokenizer(text, SyntaxFactory.Token(expectedTokenKind, text));

src/Compiler/Microsoft.AspNetCore.Razor.Language/test/Legacy/DirectiveCSharpTokenizerTest.cs

@@ -42,7 +42,7 @@ public void Next_IncludesComments_ReturnsNull_AfterTokenizingFirstDirective()
             SyntaxFactory.Token(SyntaxKind.NewLine, "\r\n"));
     }
 
-    internal override object CreateTokenizer(SeekableTextReader source)
+    internal override object CreateTokenizer(SeekableTextReader source, CodeAnalysis.CSharp.CSharpParseOptions parseOptions)
     {
         return new DirectiveCSharpTokenizer(source);
     }

src/Compiler/Microsoft.AspNetCore.Razor.Language/test/Legacy/DirectiveHtmlTokenizerTest.cs

@@ -36,7 +36,7 @@ public void Next_IncludesRazorComments_ReturnsNull_WhenHtmlIsSeen()
             SyntaxFactory.Token(SyntaxKind.OpenAngle, "<"));
     }
 
-    internal override object CreateTokenizer(SeekableTextReader source)
+    internal override object CreateTokenizer(SeekableTextReader source, object o)
     {
         return new DirectiveHtmlTokenizer(source);
     }

src/Compiler/Microsoft.AspNetCore.Razor.Language/test/Legacy/HtmlTokenizerTestBase.cs

@@ -3,11 +3,12 @@
 
 #nullable disable
 
+using System.Diagnostics;
 using Microsoft.AspNetCore.Razor.Language.Syntax.InternalSyntax;
 
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
-public abstract class HtmlTokenizerTestBase : TokenizerTestBase
+public abstract class HtmlTokenizerTestBase : TokenizerTestBase<object>
 {
     private static readonly SyntaxToken _ignoreRemaining = SyntaxFactory.Token(SyntaxKind.Marker, string.Empty);
 
@@ -16,8 +17,11 @@ internal override object IgnoreRemaining
         get { return _ignoreRemaining; }
     }
 
-    internal override object CreateTokenizer(SeekableTextReader source)
+    internal override object DefaultTokenizerArg => null;
+
+    internal override object CreateTokenizer(SeekableTextReader source, object tokenizerArg)
     {
+        Debug.Assert(tokenizerArg == null);
         return new HtmlTokenizer(source);
     }
 

src/Compiler/Microsoft.AspNetCore.Razor.Language/test/Legacy/TokenizerTestBase.cs

@@ -12,19 +12,25 @@
 
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
-public abstract class TokenizerTestBase
+public abstract class TokenizerTestBase<TTokenizerArg> where TTokenizerArg : class
 {
     internal abstract object IgnoreRemaining { get; }
-    internal abstract object CreateTokenizer(SeekableTextReader source);
+    internal abstract object CreateTokenizer(SeekableTextReader source, TTokenizerArg tokenizerArg);
+    internal abstract TTokenizerArg DefaultTokenizerArg { get; }
 
     internal void TestTokenizer(string input, params SyntaxToken[] expectedSymbols)
+    {
+        TestTokenizer(input, DefaultTokenizerArg, expectedSymbols);
+    }
+
+    internal void TestTokenizer(string input, TTokenizerArg tokenizerArg, params SyntaxToken[] expectedSymbols)
     {
         // Arrange
         var success = true;
         var output = new StringBuilder();
         using (var source = new SeekableTextReader(input, filePath: null))
         {
-            var tokenizer = (Tokenizer)CreateTokenizer(source);
+            var tokenizer = (Tokenizer)CreateTokenizer(source, tokenizerArg);
             var counter = 0;
             SyntaxToken current = null;
             while ((current = tokenizer.NextToken()) != null)

src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Legacy/CSharpCodeParser.cs

@@ -92,7 +92,7 @@ public CSharpCodeParser(IEnumerable<DirectiveDescriptor> directives, ParserConte
         : base(context.ParseLeadingDirectives
             ? FirstDirectiveCSharpLanguageCharacteristics.Instance
             : context.UseRoslynTokenizer
-                ? RoslynCSharpLanguageCharacteristics.Instance
+                ? new RoslynCSharpLanguageCharacteristics(context.CSharpParseOptions)
                 : NativeCSharpLanguageCharacteristics.Instance, context)
     {
         if (directives == null)

src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Legacy/ParserContext.cs

@@ -5,6 +5,7 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
+using Microsoft.CodeAnalysis.CSharp;
 
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
@@ -27,6 +28,7 @@ public ParserContext(RazorSourceDocument source, RazorParserOptions options)
         SeenDirectives = new HashSet<string>(StringComparer.Ordinal);
         EnableSpanEditHandlers = options.EnableSpanEditHandlers;
         UseRoslynTokenizer = options.UseRoslynTokenizer;
+        CSharpParseOptions = options.CSharpParseOptions;
     }
 
     public ErrorSink ErrorSink { get; set; }
@@ -45,6 +47,8 @@ public ParserContext(RazorSourceDocument source, RazorParserOptions options)
 
     public bool UseRoslynTokenizer { get; }
 
+    public CSharpParseOptions CSharpParseOptions { get; }
+
     public bool EnableSpanEditHandlers { get; }
 
     public bool WhiteSpaceIsSignificantToAncestorBlock { get; set; }

src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Legacy/RoslynCSharpLanguageCharacteristics.cs

@@ -10,7 +10,7 @@
 namespace Microsoft.AspNetCore.Razor.Language.Legacy;
 
 // Removal of this type is tracked by https://github.com/dotnet/razor/issues/8445
-internal class RoslynCSharpLanguageCharacteristics : LanguageCharacteristics<CSharpTokenizer>
+internal class RoslynCSharpLanguageCharacteristics(CodeAnalysis.CSharp.CSharpParseOptions csharpParseOptions) : LanguageCharacteristics<CSharpTokenizer>
 {
     private static readonly Dictionary<SyntaxKind, string> _tokenSamples = new Dictionary<SyntaxKind, string>()
         {
@@ -65,17 +65,9 @@ internal class RoslynCSharpLanguageCharacteristics : LanguageCharacteristics<CSh
             { SyntaxKind.Transition, "@" },
         };
 
-    private static readonly RoslynCSharpLanguageCharacteristics _instance = new RoslynCSharpLanguageCharacteristics();
-
-    protected RoslynCSharpLanguageCharacteristics()
-    {
-    }
-
-    public static RoslynCSharpLanguageCharacteristics Instance => _instance;
-
     public override CSharpTokenizer CreateTokenizer(SeekableTextReader source)
     {
-        return new RoslynCSharpTokenizer(source);
+        return new RoslynCSharpTokenizer(source, csharpParseOptions);
     }
 
     protected override SyntaxToken CreateToken(string content, SyntaxKind kind, RazorDiagnostic[] errors)

src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/Legacy/RoslynCSharpTokenizer.cs

@@ -33,13 +33,12 @@ internal sealed class RoslynCSharpTokenizer : CSharpTokenizer
     /// </summary>
     private readonly List<(int position, SyntaxTokenParser.Result result)> _resultCache = ListPool<(int, SyntaxTokenParser.Result)>.Default.Get();
 
-    public RoslynCSharpTokenizer(SeekableTextReader source)
+    public RoslynCSharpTokenizer(SeekableTextReader source, CSharpParseOptions parseOptions)
         : base(source)
     {
         base.CurrentState = StartState;
 
-        // PROTOTYPE
-        _roslynTokenParser = CodeAnalysis.CSharp.SyntaxFactory.CreateTokenParser(source.SourceText, null);
+        _roslynTokenParser = CodeAnalysis.CSharp.SyntaxFactory.CreateTokenParser(source.SourceText, parseOptions);
     }
 
     protected override int StartState => (int)RoslynCSharpTokenizerState.Start;

src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/RazorParserOptions.cs

@@ -5,6 +5,7 @@
 
 using System;
 using System.Collections.Generic;
+using Microsoft.CodeAnalysis.CSharp;
 
 namespace Microsoft.AspNetCore.Razor.Language;
 
@@ -19,7 +20,8 @@ public static RazorParserOptions CreateDefault()
             useRoslynTokenizer: false,
             version: RazorLanguageVersion.Latest,
             fileKind: FileKinds.Legacy,
-            enableSpanEditHandlers: false);
+            enableSpanEditHandlers: false,
+            csharpParseOptions: CSharpParseOptions.Default);
     }
 
     public static RazorParserOptions Create(Action<RazorParserOptionsBuilder> configure)
@@ -60,7 +62,7 @@ public static RazorParserOptions CreateDesignTime(Action<RazorParserOptionsBuild
         return options;
     }
 
-    internal RazorParserOptions(DirectiveDescriptor[] directives, bool designTime, bool parseLeadingDirectives, bool useRoslynTokenizer, RazorLanguageVersion version, string fileKind, bool enableSpanEditHandlers)
+    internal RazorParserOptions(DirectiveDescriptor[] directives, bool designTime, bool parseLeadingDirectives, bool useRoslynTokenizer, RazorLanguageVersion version, string fileKind, bool enableSpanEditHandlers, CSharpParseOptions csharpParseOptions)
     {
         if (directives == null)
         {
@@ -80,6 +82,7 @@ internal RazorParserOptions(DirectiveDescriptor[] directives, bool designTime, b
         FeatureFlags = RazorParserFeatureFlags.Create(Version, fileKind);
         FileKind = fileKind;
         EnableSpanEditHandlers = enableSpanEditHandlers;
+        CSharpParseOptions = csharpParseOptions;
     }
 
     public bool DesignTime { get; }
@@ -98,6 +101,8 @@ internal RazorParserOptions(DirectiveDescriptor[] directives, bool designTime, b
 
     public bool UseRoslynTokenizer { get; }
 
+    public CSharpParseOptions CSharpParseOptions { get; }
+
     public RazorLanguageVersion Version { get; } = RazorLanguageVersion.Latest;
 
     internal string FileKind { get; }

src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/RazorParserOptionsBuilder.cs

@@ -6,6 +6,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using Microsoft.CodeAnalysis.CSharp;
 
 namespace Microsoft.AspNetCore.Razor.Language;
 
@@ -44,13 +45,15 @@ internal RazorParserOptionsBuilder(bool designTime, RazorLanguageVersion version
 
     public bool UseRoslynTokenizer { get; set; }
 
+    public CSharpParseOptions CSharpParseOptions { get; set; }
+
     public RazorLanguageVersion LanguageVersion { get; }
 
     internal bool EnableSpanEditHandlers { get; set; }
 
     public RazorParserOptions Build()
     {
-        return new RazorParserOptions(Directives.ToArray(), DesignTime, ParseLeadingDirectives, UseRoslynTokenizer, LanguageVersion, FileKind ?? FileKinds.Legacy, EnableSpanEditHandlers);
+        return new RazorParserOptions(Directives.ToArray(), DesignTime, ParseLeadingDirectives, UseRoslynTokenizer, LanguageVersion, FileKind ?? FileKinds.Legacy, EnableSpanEditHandlers, CSharpParseOptions);
     }
 
     public void SetDesignTime(bool designTime)

src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/SourceGenerators/ConfigureRazorParserOptions.cs

@@ -2,15 +2,17 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using Microsoft.AspNetCore.Razor.Language;
+using Microsoft.CodeAnalysis.CSharp;
 
 namespace Microsoft.NET.Sdk.Razor.SourceGenerators;
 
-internal class ConfigureRazorParserOptions(bool useRoslynTokenizer) : RazorEngineFeatureBase, IConfigureRazorParserOptionsFeature
+internal class ConfigureRazorParserOptions(bool useRoslynTokenizer, CSharpParseOptions csharpParseOptions) : RazorEngineFeatureBase, IConfigureRazorParserOptionsFeature
 {
     public int Order { get; set; }
 
     public void Configure(RazorParserOptionsBuilder options)
     {
         options.UseRoslynTokenizer = useRoslynTokenizer;
+        options.CSharpParseOptions = csharpParseOptions;
     }
 }

src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/SourceGenerators/RazorSourceGenerationOptions.cs

@@ -18,10 +18,7 @@ internal sealed record RazorSourceGenerationOptions
         /// </summary>
         public bool GenerateMetadataSourceChecksumAttributes { get; set; } = false;
 
-        /// <summary>
-        /// Gets the CSharp language version currently used by the compilation.
-        /// </summary>
-        public LanguageVersion CSharpLanguageVersion { get; set; } = LanguageVersion.CSharp10;
+        internal CSharpParseOptions CSharpParseOptions { get; set; } = new CSharpParseOptions(LanguageVersion.CSharp10);
 
         /// <summary>
         /// Gets a flag that determines if localized component names should be supported.

src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/SourceGenerators/RazorSourceGenerator.Helpers.cs

@@ -54,14 +54,14 @@ private static RazorProjectEngine GetDeclarationProjectEngine(
                     options.SuppressChecksum = true;
                     options.SupportLocalizedComponentNames = razorSourceGeneratorOptions.SupportLocalizedComponentNames;
                 }));
-                b.Features.Add(new ConfigureRazorParserOptions(razorSourceGeneratorOptions.UseRoslynTokenizer));
+                b.Features.Add(new ConfigureRazorParserOptions(razorSourceGeneratorOptions.UseRoslynTokenizer, razorSourceGeneratorOptions.CSharpParseOptions));
 
                 b.SetRootNamespace(razorSourceGeneratorOptions.RootNamespace);
 
                 CompilerFeatures.Register(b);
                 RazorExtensions.Register(b);
 
-                b.SetCSharpLanguageVersion(razorSourceGeneratorOptions.CSharpLanguageVersion);
+                b.SetCSharpLanguageVersion(razorSourceGeneratorOptions.CSharpParseOptions.LanguageVersion);
             });
 
             return discoveryProjectEngine;
@@ -109,12 +109,12 @@ private static SourceGeneratorProjectEngine GetGenerationProjectEngine(
                     options.SuppressUniqueIds = razorSourceGeneratorOptions.TestSuppressUniqueIds;
                     options.SuppressAddComponentParameter = !isAddComponentParameterAvailable;
                 }));
-                b.Features.Add(new ConfigureRazorParserOptions(razorSourceGeneratorOptions.UseRoslynTokenizer));
+                b.Features.Add(new ConfigureRazorParserOptions(razorSourceGeneratorOptions.UseRoslynTokenizer, razorSourceGeneratorOptions.CSharpParseOptions));
 
                 CompilerFeatures.Register(b);
                 RazorExtensions.Register(b);
 
-                b.SetCSharpLanguageVersion(razorSourceGeneratorOptions.CSharpLanguageVersion);
+                b.SetCSharpLanguageVersion(razorSourceGeneratorOptions.CSharpParseOptions.LanguageVersion);
             });
 
             return new SourceGeneratorProjectEngine(projectEngine);

src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/SourceGenerators/RazorSourceGenerator.RazorProviders.cs

@@ -54,7 +54,7 @@ public partial class RazorSourceGenerator
                 GenerateMetadataSourceChecksumAttributes = generateMetadataSourceChecksumAttributes == "true",
                 RootNamespace = rootNamespace ?? "ASP",
                 SupportLocalizedComponentNames = supportLocalizedComponentNames == "true",
-                CSharpLanguageVersion = ((CSharpParseOptions)parseOptions).LanguageVersion,
+                CSharpParseOptions = (CSharpParseOptions)parseOptions,
                 TestSuppressUniqueIds = _testSuppressUniqueIds,
                 UseRoslynTokenizer = useRazorTokenizer,
             };

src/Razor/test/Microsoft.AspNetCore.Razor.Test.Common.Tooling/Language/Legacy/ToolingParserTestBase.cs

@@ -10,6 +10,7 @@
 using System.Reflection;
 using System.Text.RegularExpressions;
 using Microsoft.AspNetCore.Razor.Test.Common;
+using Microsoft.CodeAnalysis.CSharp;
 using Roslyn.Test.Utilities;
 using Xunit;
 using Xunit.Abstractions;
@@ -253,10 +254,11 @@ internal static RazorParserOptions CreateParserOptions(
             directives.ToArray(),
             designTime,
             parseLeadingDirectives: false,
-            useRoslynTokenizer: false,
+            useRoslynTokenizer: false, // PROTOTYPE: switch to true
             version: version,
             fileKind: fileKind,
-            enableSpanEditHandlers)
+            enableSpanEditHandlers,
+            csharpParseOptions: CSharpParseOptions.Default)
             {
                 FeatureFlags = featureFlags ?? RazorParserFeatureFlags.Create(version, fileKind)
             };

src/Shared/Microsoft.AspNetCore.Razor.Test.Common/Language/IntegrationTests/IntegrationTestBase.cs

@@ -318,7 +318,7 @@ private RazorProjectEngine CreateProjectEngine(RazorConfiguration configuration,
 
             b.Features.Add(new DefaultTypeNameFeature());
             b.SetCSharpLanguageVersion(CSharpParseOptions.LanguageVersion);
-            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true));
+            b.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer: true, CSharpParseOptions));
 
             // Decorate each import feature so we can normalize line endings.
             foreach (var feature in b.Features.OfType<IImportProjectFeature>().ToArray())