Update project configuration from Roslyn info (dotnet#11092)

Fixes dotnet#10736 and brings in IDE
support for the new Roslyn tokenizer, which makes this work:


![Image](https://github.com/user-attachments/assets/3f3647d3-5089-4750-96a0-a80c498db97b)

src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/CSharp/ParseOptionsExtensions.cs

@@ -0,0 +1,13 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+
+namespace Microsoft.CodeAnalysis.Razor.Compiler.CSharp;
+
+internal static class ParseOptionsExtensions
+{
+    public static bool UseRoslynTokenizer(this ParseOptions parseOptions)
+        => parseOptions.Features.TryGetValue("use-roslyn-tokenizer", out var useRoslynTokenizerValue)
+           && string.Equals(useRoslynTokenizerValue, "true", StringComparison.OrdinalIgnoreCase);
+}

src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/Language/RazorConfiguration.cs

@@ -3,6 +3,7 @@
 
 using System.Collections.Immutable;
 using System.Linq;
+using Microsoft.CodeAnalysis.CSharp;
 using Microsoft.Extensions.Internal;
 
 namespace Microsoft.AspNetCore.Razor.Language;
@@ -13,15 +14,14 @@ public sealed record class RazorConfiguration(
     ImmutableArray<RazorExtension> Extensions,
     bool UseConsolidatedMvcViews = true,
     bool SuppressAddComponentParameter = false,
-    LanguageServerFlags? LanguageServerFlags = null)
+    LanguageServerFlags? LanguageServerFlags = null,
+    bool UseRoslynTokenizer = false,
+    LanguageVersion CSharpLanguageVersion = LanguageVersion.Default)
 {
     public static readonly RazorConfiguration Default = new(
         RazorLanguageVersion.Latest,
         ConfigurationName: "unnamed",
-        Extensions: [],
-        UseConsolidatedMvcViews: true,
-        SuppressAddComponentParameter: false,
-        LanguageServerFlags: null);
+        Extensions: []);
 
     public bool Equals(RazorConfiguration? other)
         => other is not null &&
@@ -30,6 +30,8 @@ public bool Equals(RazorConfiguration? other)
            SuppressAddComponentParameter == other.SuppressAddComponentParameter &&
            LanguageServerFlags == other.LanguageServerFlags &&
            UseConsolidatedMvcViews == other.UseConsolidatedMvcViews &&
+           UseRoslynTokenizer == other.UseRoslynTokenizer &&
+           CSharpLanguageVersion == other.CSharpLanguageVersion &&
            Extensions.SequenceEqual(other.Extensions);
 
     public override int GetHashCode()
@@ -41,6 +43,8 @@ public override int GetHashCode()
         hash.Add(SuppressAddComponentParameter);
         hash.Add(UseConsolidatedMvcViews);
         hash.Add(LanguageServerFlags);
+        hash.Add(UseRoslynTokenizer);
+        hash.Add(CSharpLanguageVersion);
         return hash;
     }
 }

src/Compiler/Microsoft.CodeAnalysis.Razor.Compiler/src/SourceGenerators/RazorSourceGenerator.RazorProviders.cs

@@ -56,8 +56,7 @@ public partial class RazorSourceGenerator
             var razorConfiguration = new RazorConfiguration(razorLanguageVersion, configurationName ?? "default", Extensions: [], UseConsolidatedMvcViews: true, SuppressAddComponentParameter: !isComponentParameterSupported);
 
             // We use the new tokenizer only when requested for now.
-            var useRoslynTokenizer = parseOptions.Features.TryGetValue("use-roslyn-tokenizer", out var useRoslynTokenizerValue)
-                                    && string.Equals(useRoslynTokenizerValue, "true", StringComparison.OrdinalIgnoreCase);
+            var useRoslynTokenizer = parseOptions.UseRoslynTokenizer();
 
             var razorSourceGenerationOptions = new RazorSourceGenerationOptions()
             {

src/Razor/benchmarks/Microsoft.AspNetCore.Razor.Microbenchmarks/LanguageServer/RazorLanguageServerBenchmarkBase.cs

@@ -75,7 +75,7 @@ await projectManager.UpdateAsync(
             {
                 updater.ProjectAdded(hostProject);
                 var tagHelpers = CommonResources.LegacyTagHelpers;
-                var projectWorkspaceState = ProjectWorkspaceState.Create(tagHelpers, CodeAnalysis.CSharp.LanguageVersion.CSharp11);
+                var projectWorkspaceState = ProjectWorkspaceState.Create(tagHelpers);
                 updater.ProjectWorkspaceStateChanged(hostProject.Key, projectWorkspaceState);
                 updater.DocumentAdded(hostProject.Key, hostDocument, textLoader);
             },

src/Razor/benchmarks/Microsoft.AspNetCore.Razor.Microbenchmarks/Resources/Telerik/Kendo.Mvc.Examples.project.razor.json

@@ -1,10 +1,12 @@
 {
-  "__Version": 6,
+  "__Version": 7,
   "ProjectKey": "C:\\Users\\admin\\location\\Kendo.Mvc.Examples\\obj\\Debug\\net7.0\\",
   "FilePath": "C:\\Users\\admin\\location\\Kendo.Mvc.Examples\\Kendo.Mvc.Examples.csproj",
   "Configuration": {
     "ConfigurationName": "MVC-3.0",
     "LanguageVersion": "7.0",
+    "UseRoslynTokenizer": true,
+    "CSharpLanguageVersion": 1100,
     "Extensions": [
       "MVC-3.0"
     ]
@@ -159526,8 +159528,7 @@
           "Runtime.Name": "Components.None"
         }
       }
-    ],
-    "CSharpLanguageVersion": 1100
+    ]
   },
   "RootNamespace": "Kendo.Mvc.Examples",
   "Documents": [

src/Razor/benchmarks/Microsoft.AspNetCore.Razor.Microbenchmarks/Resources/project.razor.json

@@ -1,10 +1,12 @@
 {
-  "__Version": 6,
+  "__Version": 7,
   "ProjectKey": "C:\\Users\\admin\\location\\blazorserver\\obj\\Debug\\net7.0\\",
   "FilePath": "C:\\Users\\admin\\location\\blazorserver\\blazorserver.csproj",
   "Configuration": {
     "ConfigurationName": "MVC-3.0",
     "LanguageVersion": "3.0",
+    "UseRoslynTokenizer": true,
+    "CSharpLanguageVersion": 800,
     "Extensions": [ "MVC-3.0" ]
   },
   "ProjectWorkspaceState": {
@@ -16125,8 +16127,7 @@
           "Runtime.Name": "Components.None"
         }
       }
-    ],
-    "CSharpLanguageVersion": 800
+    ]
   },
   "RootNamespace": "blazorserver",
   "Documents": [

src/Razor/src/Microsoft.AspNetCore.Razor.LanguageServer/ProjectSystem/RazorProjectService.cs

@@ -366,17 +366,16 @@ private Task AddOrUpdateProjectCoreAsync(
 
                 if (!projectWorkspaceState.Equals(ProjectWorkspaceState.Default))
                 {
-                    _logger.LogInformation($"Updating project '{project.Key}' TagHelpers ({projectWorkspaceState.TagHelpers.Length}) and C# Language Version ({projectWorkspaceState.CSharpLanguageVersion}).");
+                    _logger.LogInformation($"Updating project '{project.Key}' TagHelpers ({projectWorkspaceState.TagHelpers.Length}).");
                 }
 
                 updater.ProjectWorkspaceStateChanged(project.Key, projectWorkspaceState);
 
-                var currentConfiguration = project.Configuration;
                 var currentRootNamespace = project.RootNamespace;
-                if (currentConfiguration.ConfigurationName == configuration?.ConfigurationName &&
+                if (project.Configuration == configuration &&
                     currentRootNamespace == rootNamespace)
                 {
-                    _logger.LogTrace($"Updating project '{project.Key}'. The project is already using configuration '{configuration.ConfigurationName}' and root namespace '{rootNamespace}'.");
+                    _logger.LogTrace($"Skipping configuration update for '{project.Key}' as the configuration and root namespace are unchanged.");
                     return;
                 }
 
@@ -385,14 +384,9 @@ private Task AddOrUpdateProjectCoreAsync(
                     configuration = FallbackRazorConfiguration.Latest;
                     _logger.LogInformation($"Updating project '{project.Key}' to use the latest configuration ('{configuration.ConfigurationName}')'.");
                 }
-                else if (currentConfiguration.ConfigurationName != configuration.ConfigurationName)
+                else
                 {
-                    _logger.LogInformation($"Updating project '{project.Key}' to Razor configuration '{configuration.ConfigurationName}' with language version '{configuration.LanguageVersion}'.");
-                }
-
-                if (currentRootNamespace != rootNamespace)
-                {
-                    _logger.LogInformation($"Updating project '{project.Key}''s root namespace to '{rootNamespace}'.");
+                    _logger.LogInformation($"Updating project '{project.Key}' to Razor configuration '{configuration.ConfigurationName}', namespace '{rootNamespace}', with language version '{configuration.LanguageVersion}'.");
                 }
 
                 var hostProject = new HostProject(project.FilePath, project.IntermediateOutputPath, configuration, rootNamespace, displayName);

src/Razor/src/Microsoft.AspNetCore.Razor.ProjectEngineHost/ProjectSystem/ProjectWorkspaceState.cs

@@ -5,52 +5,40 @@
 using System.Collections.Immutable;
 using System.Linq;
 using Microsoft.AspNetCore.Razor.Language;
-using Microsoft.CodeAnalysis.CSharp;
 using Microsoft.Extensions.Internal;
 
 namespace Microsoft.AspNetCore.Razor.ProjectSystem;
 
 internal sealed class ProjectWorkspaceState : IEquatable<ProjectWorkspaceState>
 {
-    public static readonly ProjectWorkspaceState Default = new(ImmutableArray<TagHelperDescriptor>.Empty, LanguageVersion.Default);
+    public static readonly ProjectWorkspaceState Default = new(ImmutableArray<TagHelperDescriptor>.Empty);
 
     public ImmutableArray<TagHelperDescriptor> TagHelpers { get; }
-    public LanguageVersion CSharpLanguageVersion { get; }
 
     private ProjectWorkspaceState(
-        ImmutableArray<TagHelperDescriptor> tagHelpers,
-        LanguageVersion csharpLanguageVersion)
+        ImmutableArray<TagHelperDescriptor> tagHelpers)
     {
         TagHelpers = tagHelpers;
-        CSharpLanguageVersion = csharpLanguageVersion;
     }
 
     public static ProjectWorkspaceState Create(
-        ImmutableArray<TagHelperDescriptor> tagHelpers,
-        LanguageVersion csharpLanguageVersion = LanguageVersion.Default)
-        => tagHelpers.IsEmpty && csharpLanguageVersion == LanguageVersion.Default
+        ImmutableArray<TagHelperDescriptor> tagHelpers)
+        => tagHelpers.IsEmpty
             ? Default
-            : new(tagHelpers, csharpLanguageVersion);
-
-    public static ProjectWorkspaceState Create(LanguageVersion csharpLanguageVersion)
-        => csharpLanguageVersion == LanguageVersion.Default
-            ? Default
-            : new(ImmutableArray<TagHelperDescriptor>.Empty, csharpLanguageVersion);
+            : new(tagHelpers);
 
     public override bool Equals(object? obj)
         => obj is ProjectWorkspaceState other && Equals(other);
 
     public bool Equals(ProjectWorkspaceState? other)
         => other is not null &&
-           TagHelpers.SequenceEqual(other.TagHelpers) &&
-           CSharpLanguageVersion == other.CSharpLanguageVersion;
+           TagHelpers.SequenceEqual(other.TagHelpers);
 
     public override int GetHashCode()
     {
         var hash = HashCodeCombiner.Start();
 
         hash.Add(TagHelpers);
-        hash.Add(CSharpLanguageVersion);
 
         return hash.CombinedHash;
     }

src/Razor/src/Microsoft.AspNetCore.Razor.ProjectEngineHost/Serialization/MessagePack/Formatters/ProjectWorkspaceStateFormatter.cs

@@ -8,7 +8,6 @@
 using Microsoft.AspNetCore.Razor.ProjectSystem;
 using Microsoft.AspNetCore.Razor.Serialization.MessagePack.Formatters.TagHelpers;
 using Microsoft.AspNetCore.Razor.Utilities;
-using Microsoft.CodeAnalysis.CSharp;
 
 namespace Microsoft.AspNetCore.Razor.Serialization.MessagePack.Formatters;
 
@@ -22,7 +21,7 @@ private ProjectWorkspaceStateFormatter()
 
     public override ProjectWorkspaceState Deserialize(ref MessagePackReader reader, SerializerCachingOptions options)
     {
-        reader.ReadArrayHeaderAndVerify(3);
+        reader.ReadArrayHeaderAndVerify(2);
 
         var checksums = reader.Deserialize<ImmutableArray<Checksum>>(options);
 
@@ -47,19 +46,17 @@ public override ProjectWorkspaceState Deserialize(ref MessagePackReader reader,
         }
 
         var tagHelpers = builder.DrainToImmutable();
-        var csharpLanguageVersion = (LanguageVersion)reader.ReadInt32();
 
-        return ProjectWorkspaceState.Create(tagHelpers, csharpLanguageVersion);
+        return ProjectWorkspaceState.Create(tagHelpers);
     }
 
     public override void Serialize(ref MessagePackWriter writer, ProjectWorkspaceState value, SerializerCachingOptions options)
     {
-        writer.WriteArrayHeader(3);
+        writer.WriteArrayHeader(2);
 
         var checksums = value.TagHelpers.SelectAsArray(x => x.Checksum);
 
         writer.Serialize(checksums, options);
         writer.Serialize(value.TagHelpers, options);
-        writer.Write((int)value.CSharpLanguageVersion);
     }
 }

src/Razor/src/Microsoft.AspNetCore.Razor.ProjectEngineHost/Serialization/MessagePack/Formatters/RazorConfigurationFormatter.cs

@@ -4,13 +4,18 @@
 using MessagePack;
 using Microsoft.AspNetCore.Razor.Language;
 using Microsoft.AspNetCore.Razor.PooledObjects;
+using Microsoft.CodeAnalysis.CSharp;
 
 namespace Microsoft.AspNetCore.Razor.Serialization.MessagePack.Formatters;
 
 internal sealed class RazorConfigurationFormatter : ValueFormatter<RazorConfiguration>
 {
     public static readonly ValueFormatter<RazorConfiguration> Instance = new RazorConfigurationFormatter();
 
+    // The count of properties in RazorConfiguration that are serialized. The number of Extensions will be added
+    // to this, for the final serialized value count.
+    private const int SerializedPropertyCount = 6;
+
     private RazorConfigurationFormatter()
     {
     }
@@ -24,8 +29,10 @@ public override RazorConfiguration Deserialize(ref MessagePackReader reader, Ser
         var languageVersionText = CachedStringFormatter.Instance.Deserialize(ref reader, options) ?? string.Empty;
         var suppressAddComponentParameter = reader.ReadBoolean();
         var useConsolidatedMvcViews = reader.ReadBoolean();
+        var useRoslynTokenizer = reader.ReadBoolean();
+        var csharpLanguageVersion = (LanguageVersion)reader.ReadInt32();
 
-        count -= 4;
+        count -= SerializedPropertyCount;
 
         using var builder = new PooledArrayBuilder<RazorExtension>();
 
@@ -46,14 +53,16 @@ public override RazorConfiguration Deserialize(ref MessagePackReader reader, Ser
             configurationName,
             extensions,
             UseConsolidatedMvcViews: useConsolidatedMvcViews,
-            SuppressAddComponentParameter: suppressAddComponentParameter);
+            SuppressAddComponentParameter: suppressAddComponentParameter,
+            UseRoslynTokenizer: useRoslynTokenizer,
+            CSharpLanguageVersion: csharpLanguageVersion);
     }
 
     public override void Serialize(ref MessagePackWriter writer, RazorConfiguration value, SerializerCachingOptions options)
     {
-        // Write 4 values + 1 value per extension.
+        // Write SerializedPropertyCount values + 1 value per extension.
         var extensions = value.Extensions;
-        var count = extensions.Length + 4;
+        var count = extensions.Length + SerializedPropertyCount;
 
         writer.WriteArrayHeader(count);
 
@@ -70,8 +79,10 @@ public override void Serialize(ref MessagePackWriter writer, RazorConfiguration
 
         writer.Write(value.SuppressAddComponentParameter);
         writer.Write(value.UseConsolidatedMvcViews);
+        writer.Write(value.UseRoslynTokenizer);
+        writer.Write((int)value.CSharpLanguageVersion);
 
-        count -= 4;
+        count -= SerializedPropertyCount;
 
         for (var i = 0; i < count; i++)
         {

src/Razor/src/Microsoft.AspNetCore.Razor.ProjectEngineHost/Serialization/MessagePack/SerializationFormat.cs

@@ -9,5 +9,5 @@ internal static class SerializationFormat
     // or any of the types that compose it changes. This includes: RazorConfiguration,
     // ProjectWorkspaceState, TagHelperDescriptor, and DocumentSnapshotHandle.
     // NOTE: If this version is changed, a coordinated insertion is required between Roslyn and Razor for the C# extension.
-    public const int Version = 6;
+    public const int Version = 7;
 }

src/Razor/src/Microsoft.AspNetCore.Razor.ProjectEngineHost/Utilities/RazorProjectInfoFactory.cs

@@ -19,6 +19,7 @@
 using Microsoft.CodeAnalysis.Diagnostics;
 using Microsoft.CodeAnalysis.Razor;
 using Microsoft.CodeAnalysis.Razor.Compiler.CSharp;
+using Microsoft.NET.Sdk.Razor.SourceGenerators;
 
 namespace Microsoft.AspNetCore.Razor.Utilities;
 
@@ -56,7 +57,9 @@ static RazorProjectInfoFactory()
             return null;
         }
 
-        var csharpLanguageVersion = (project.ParseOptions as CSharpParseOptions)?.LanguageVersion ?? LanguageVersion.Default;
+        var csharpParseOptions = project.ParseOptions as CSharpParseOptions ?? CSharpParseOptions.Default;
+        var csharpLanguageVersion = csharpParseOptions.LanguageVersion;
+        var useRoslynTokenizer = csharpParseOptions.UseRoslynTokenizer();
 
         var compilation = await project.GetCompilationAsync(cancellationToken).ConfigureAwait(false);
         if (compilation is null)
@@ -77,6 +80,7 @@ static RazorProjectInfoFactory()
 
             builder.SetCSharpLanguageVersion(csharpLanguageVersion);
             builder.SetSupportLocalizedComponentNames(); // ProjectState in MS.CA.Razor.Workspaces does this, so I'm doing it too!
+            builder.Features.Add(new ConfigureRazorParserOptions(useRoslynTokenizer, csharpParseOptions));
         };
 
         var engineFactory = ProjectEngineFactories.DefaultProvider.GetFactory(configuration);
@@ -88,7 +92,7 @@ static RazorProjectInfoFactory()
 
         var tagHelpers = await project.GetTagHelpersAsync(engine, NoOpTelemetryReporter.Instance, cancellationToken).ConfigureAwait(false);
 
-        var projectWorkspaceState = ProjectWorkspaceState.Create(tagHelpers, csharpLanguageVersion);
+        var projectWorkspaceState = ProjectWorkspaceState.Create(tagHelpers);
 
         return new RazorProjectInfo(
             projectKey: new ProjectKey(intermediateOutputPath),

src/Razor/src/Microsoft.AspNetCore.Razor.ProjectEngineHost/Utilities/StreamExtensions.NetCore.cs

@@ -1,14 +1,14 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT license. See License.txt in the project root for license information.
 
-using Microsoft.AspNetCore.Razor.ProjectSystem;
 using System;
 using System.Buffers;
 using System.Diagnostics;
 using System.IO;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.AspNetCore.Razor.ProjectSystem;
 
 namespace Microsoft.AspNetCore.Razor.Utilities;