Support side by side workload manifests in SDK resolver (#32471)

src/Common/EnvironmentVariableNames.cs

@@ -14,6 +14,7 @@ static class EnvironmentVariableNames
         public static readonly string ALLOW_TARGETING_PACK_CACHING = "DOTNETSDK_ALLOW_TARGETING_PACK_CACHING";
         public static readonly string WORKLOAD_PACK_ROOTS = "DOTNETSDK_WORKLOAD_PACK_ROOTS";
         public static readonly string WORKLOAD_MANIFEST_ROOTS = "DOTNETSDK_WORKLOAD_MANIFEST_ROOTS";
+        public static readonly string WORKLOAD_MANIFEST_IGNORE_DEFAULT_ROOTS = "DOTNETSDK_WORKLOAD_MANIFEST_IGNORE_DEFAULT_ROOTS";
         public static readonly string WORKLOAD_UPDATE_NOTIFY_DISABLE = "DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE";
         public static readonly string WORKLOAD_UPDATE_NOTIFY_INTERVAL_HOURS = "DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_INTERVAL_HOURS";
         public static readonly string WORKLOAD_DISABLE_PACK_GROUPS = "DOTNET_CLI_WORKLOAD_DISABLE_PACK_GROUPS";

src/Resolvers/Microsoft.NET.Sdk.WorkloadMSBuildSdkResolver/CachingWorkloadResolver.cs

@@ -215,12 +215,3 @@ public ResolutionResult Resolve(string sdkReferenceName, string dotnetRootPath,
         }
     }
 }
-
-
-//  Add attribute to support init-only properties on .NET Framework
-#if !NET
-namespace System.Runtime.CompilerServices
-{
-    public class IsExternalInit { }
-}
-#endif

src/Resolvers/Microsoft.NET.Sdk.WorkloadManifestReader/ManifestSpecifier.cs

@@ -0,0 +1,27 @@
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+//
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Microsoft.NET.Sdk.WorkloadManifestReader
+{
+    public record class ManifestSpecifier(ManifestId Id, ManifestVersion Version, SdkFeatureBand FeatureBand)
+    {
+        public override string ToString() => $"{Id}: {Version}/{FeatureBand}";
+    }
+}
+
+
+
+//  Add attribute to support init-only properties on .NET Framework
+#if !NET
+namespace System.Runtime.CompilerServices
+{
+    public class IsExternalInit { }
+}
+#endif

src/Resolvers/Microsoft.NET.Sdk.WorkloadManifestReader/SdkDirectoryWorkloadManifestProvider.cs

@@ -6,27 +6,31 @@
 using System.ComponentModel.DataAnnotations;
 using System.IO;
 using System.Linq;
+using Microsoft.Deployment.DotNet.Releases;
 using Microsoft.DotNet.Cli;
 using Microsoft.DotNet.Workloads.Workload;
+using Microsoft.NET.Sdk.Localization;
 
 namespace Microsoft.NET.Sdk.WorkloadManifestReader
 {
     public class SdkDirectoryWorkloadManifestProvider : IWorkloadManifestProvider
     {
         private readonly string _sdkRootPath;
         private readonly SdkFeatureBand _sdkVersionBand;
-        private readonly string [] _manifestDirectories;
+        private readonly string[] _manifestRoots;
         private static HashSet<string> _outdatedManifestIds = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { "microsoft.net.workload.android", "microsoft.net.workload.blazorwebassembly", "microsoft.net.workload.ios",
             "microsoft.net.workload.maccatalyst", "microsoft.net.workload.macos", "microsoft.net.workload.tvos", "microsoft.net.workload.mono.toolchain" };
         private readonly Dictionary<string, int>? _knownManifestIdsAndOrder;
 
-        public SdkDirectoryWorkloadManifestProvider(string sdkRootPath, string sdkVersion, string? userProfileDir)
-            : this(sdkRootPath, sdkVersion, Environment.GetEnvironmentVariable, userProfileDir)
+        private readonly Dictionary<string, ManifestSpecifier> _requestedManifestVersions;
+
+        public SdkDirectoryWorkloadManifestProvider(string sdkRootPath, string sdkVersion, string? userProfileDir, IEnumerable<ManifestSpecifier>? requestedManifestVersions = null)
+            : this(sdkRootPath, sdkVersion, Environment.GetEnvironmentVariable, userProfileDir, requestedManifestVersions)
         {
 
         }
 
-        internal SdkDirectoryWorkloadManifestProvider(string sdkRootPath, string sdkVersion, Func<string, string?> getEnvironmentVariable, string? userProfileDir)
+        internal SdkDirectoryWorkloadManifestProvider(string sdkRootPath, string sdkVersion, Func<string, string?> getEnvironmentVariable, string? userProfileDir, IEnumerable<ManifestSpecifier>? requestedManifestVersions = null)
         {
             if (string.IsNullOrWhiteSpace(sdkVersion))
             {
@@ -58,25 +62,40 @@ internal SdkDirectoryWorkloadManifestProvider(string sdkRootPath, string sdkVers
                 }
             }
 
-            string? userManifestsDir = userProfileDir is null ? null : Path.Combine(userProfileDir, "sdk-manifests", _sdkVersionBand.ToString());
-            string dotnetManifestDir = Path.Combine(_sdkRootPath, "sdk-manifests", _sdkVersionBand.ToString());
-            if (userManifestsDir != null && WorkloadFileBasedInstall.IsUserLocal(_sdkRootPath, _sdkVersionBand.ToString()) && Directory.Exists(userManifestsDir))
-            {
-                _manifestDirectories = new[] { userManifestsDir, dotnetManifestDir };
-            }
-            else
+            if (getEnvironmentVariable(EnvironmentVariableNames.WORKLOAD_MANIFEST_IGNORE_DEFAULT_ROOTS) == null)
             {
-                _manifestDirectories = new[] { dotnetManifestDir };
+                string? userManifestsRoot = userProfileDir is null ? null : Path.Combine(userProfileDir, "sdk-manifests");
+                string dotnetManifestRoot = Path.Combine(_sdkRootPath, "sdk-manifests");
+                if (userManifestsRoot != null && WorkloadFileBasedInstall.IsUserLocal(_sdkRootPath, _sdkVersionBand.ToString()) && Directory.Exists(userManifestsRoot))
+                {
+                    _manifestRoots = new[] { userManifestsRoot, dotnetManifestRoot };
+                }
+                else
+                {
+                    _manifestRoots = new[] { dotnetManifestRoot };
+                }
             }
 
             var manifestDirectoryEnvironmentVariable = getEnvironmentVariable(EnvironmentVariableNames.WORKLOAD_MANIFEST_ROOTS);
             if (manifestDirectoryEnvironmentVariable != null)
             {
                 //  Append the SDK version band to each manifest root specified via the environment variable.  This allows the same
                 //  environment variable settings to be shared by multiple SDKs.
-                _manifestDirectories = manifestDirectoryEnvironmentVariable.Split(Path.PathSeparator)
-                    .Select(p => Path.Combine(p, _sdkVersionBand.ToString()))
-                    .Concat(_manifestDirectories).ToArray();
+                _manifestRoots = manifestDirectoryEnvironmentVariable.Split(Path.PathSeparator)
+                                    .Concat(_manifestRoots ?? Array.Empty<string>()).ToArray();
+
+            }
+
+            _manifestRoots = _manifestRoots ?? Array.Empty<string>();
+
+            _requestedManifestVersions = new Dictionary<string, ManifestSpecifier>(StringComparer.OrdinalIgnoreCase);
+
+            if (requestedManifestVersions != null)
+            {
+                foreach (var manifestVersion in requestedManifestVersions)
+                {
+                    _requestedManifestVersions[manifestVersion.Id.ToString()] = manifestVersion;
+                }
             }
         }
 
@@ -98,30 +117,40 @@ public IEnumerable<ReadableWorkloadManifest> GetManifests()
 
         public IEnumerable<string> GetManifestDirectories()
         {
+            //  Scan manifest directories
             var manifestIdsToDirectories = new Dictionary<string, string>();
-            if (_manifestDirectories.Length == 1)
+
+            void ProbeDirectory(string manifestDirectory)
+            {
+                (string? id, string? finalManifestDirectory) = ResolveManifestDirectory(manifestDirectory);
+                if (id != null && finalManifestDirectory != null)
+                {
+                    manifestIdsToDirectories.Add(id, finalManifestDirectory);
+                }
+            }
+
+            if (_manifestRoots.Length == 1)
             {
                 //  Optimization for common case where test hook to add additional directories isn't being used
-                if (Directory.Exists(_manifestDirectories[0]))
+                var manifestVersionBandDirectory = Path.Combine(_manifestRoots[0], _sdkVersionBand.ToString());
+                if (Directory.Exists(manifestVersionBandDirectory))
                 {
-                    foreach (var workloadManifestDirectory in Directory.EnumerateDirectories(_manifestDirectories[0]))
+                    foreach (var workloadManifestDirectory in Directory.EnumerateDirectories(manifestVersionBandDirectory))
                     {
-                        if (!IsManifestIdOutdated(workloadManifestDirectory))
-                        {
-                            manifestIdsToDirectories.Add(Path.GetFileName(workloadManifestDirectory), workloadManifestDirectory);
-                        }
+                        ProbeDirectory(workloadManifestDirectory);
                     }
                 }
             }
             else
             {
                 //  If the same folder name is in multiple of the workload manifest directories, take the first one
                 Dictionary<string, string> directoriesWithManifests = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
-                foreach (var manifestDirectory in _manifestDirectories.Reverse())
+                foreach (var manifestRoot in _manifestRoots.Reverse())
                 {
-                    if (Directory.Exists(manifestDirectory))
+                    var manifestVersionBandDirectory = Path.Combine(manifestRoot, _sdkVersionBand.ToString());
+                    if (Directory.Exists(manifestVersionBandDirectory))
                     {
-                        foreach (var workloadManifestDirectory in Directory.EnumerateDirectories(manifestDirectory))
+                        foreach (var workloadManifestDirectory in Directory.EnumerateDirectories(manifestVersionBandDirectory))
                         {
                             directoriesWithManifests[Path.GetFileName(workloadManifestDirectory)] = workloadManifestDirectory;
                         }
@@ -130,13 +159,16 @@ public IEnumerable<string> GetManifestDirectories()
 
                 foreach (var workloadManifestDirectory in directoriesWithManifests.Values)
                 {
-                    if (!IsManifestIdOutdated(workloadManifestDirectory))
-                    {
-                        manifestIdsToDirectories.Add(Path.GetFileName(workloadManifestDirectory), workloadManifestDirectory);
-                    }
+                    ProbeDirectory(workloadManifestDirectory);
                 }
             }
 
+            //  Load manifests that were explicitly specified
+            foreach (var kvp in _requestedManifestVersions)
+            {
+                manifestIdsToDirectories.Add(kvp.Key, GetManifestDirectoryFromSpecifier(kvp.Value));
+            }
+
             if (_knownManifestIdsAndOrder != null && _knownManifestIdsAndOrder.Keys.Any(id => !manifestIdsToDirectories.ContainsKey(id)))
             {
                 var missingManifestIds = _knownManifestIdsAndOrder.Keys.Where(id => !manifestIdsToDirectories.ContainsKey(id));
@@ -167,9 +199,46 @@ public IEnumerable<string> GetManifestDirectories()
                 .ToList();
         }
 
+        /// <summary>
+        /// Given a folder that may directly include a WorkloadManifest.json file, or may have the workload manifests in version subfolders, choose the directory
+        /// with the latest workload manifest.
+        /// </summary>
+        private (string? id, string? manifestDirectory) ResolveManifestDirectory(string manifestDirectory)
+        {
+            string manifestId = Path.GetFileName(manifestDirectory);
+            if (_outdatedManifestIds.Contains(manifestId))
+            {
+                return (null, null);
+            }
+
+            var manifestVersionDirectories = Directory.GetDirectories(manifestDirectory)
+                    .Where(dir => File.Exists(Path.Combine(dir, "WorkloadManifest.json")))
+                    .Select(dir =>
+                    {
+                        ReleaseVersion? releaseVersion = null;
+                        ReleaseVersion.TryParse(Path.GetFileName(dir), out releaseVersion);
+                        return (directory: dir, version: releaseVersion);
+                    })
+                    .Where(t => t.version != null)
+                    .OrderByDescending(t => t.version)
+                    .ToList();
+
+            //  Assume that if there are any versioned subfolders, they are higher manifest versions than a workload manifest directly in the specified folder, if it exists
+            if (manifestVersionDirectories.Any())
+            {
+                return (manifestId, manifestVersionDirectories.First().directory);
+            }
+            else if (File.Exists(Path.Combine(manifestDirectory, "WorkloadManifest.json")))
+            {
+                return (manifestId, manifestDirectory);
+            }
+            return (null, null);
+        }
+
         private string FallbackForMissingManifest(string manifestId)
         {
-            var sdkManifestPath = Path.Combine(_sdkRootPath, "sdk-manifests");
+            //  Only use the last manifest root (usually the dotnet folder itself) for fallback
+            var sdkManifestPath = _manifestRoots.Last();
             if (!Directory.Exists(sdkManifestPath))
             {
                 return string.Empty;
@@ -179,11 +248,21 @@ private string FallbackForMissingManifest(string manifestId)
                 .Select(dir => Path.GetFileName(dir))
                 .Select(featureBand => new SdkFeatureBand(featureBand))
                 .Where(featureBand => featureBand < _sdkVersionBand || _sdkVersionBand.ToStringWithoutPrerelease().Equals(featureBand.ToString(), StringComparison.Ordinal));
-            var matchingManifestFatureBands = candidateFeatureBands
-                .Where(featureBand => Directory.Exists(Path.Combine(sdkManifestPath, featureBand.ToString(), manifestId)));
-            if (matchingManifestFatureBands.Any())
+
+            var matchingManifestFatureBandsAndResolvedManifestDirectories = candidateFeatureBands
+                //  Calculate path to <FeatureBand>\<ManifestID>
+                .Select(featureBand => (featureBand, manifestDirectory: Path.Combine(sdkManifestPath, featureBand.ToString(), manifestId)))
+                //  Filter out directories that don't exist
+                .Where(t => Directory.Exists(t.manifestDirectory))
+                //  Inside directory, resolve where to find WorkloadManifest.json
+                .Select(t => (t.featureBand, res: ResolveManifestDirectory(t.manifestDirectory)))
+                //  Filter out directories where no WorkloadManifest.json was resolved
+                .Where(t => t.res.id != null && t.res.manifestDirectory != null)
+                .ToList();
+
+            if (matchingManifestFatureBandsAndResolvedManifestDirectories.Any())
             {
-                return Path.Combine(sdkManifestPath, matchingManifestFatureBands.Max()!.ToString(), manifestId);
+                return matchingManifestFatureBandsAndResolvedManifestDirectories.OrderByDescending(t => t.featureBand).First().res.manifestDirectory!;
             }
             else
             {
@@ -192,6 +271,21 @@ private string FallbackForMissingManifest(string manifestId)
             }
         }
 
+        private string GetManifestDirectoryFromSpecifier(ManifestSpecifier manifestSpecifier)
+        {
+            foreach (var manifestDirectory in _manifestRoots)
+            {
+                var specifiedManifestDirectory = Path.Combine(manifestDirectory, manifestSpecifier.FeatureBand.ToString(), manifestSpecifier.Id.ToString(),
+                    manifestSpecifier.Version.ToString());
+                if (File.Exists(Path.Combine(specifiedManifestDirectory, "WorkloadManifest.json")))
+                {
+                    return specifiedManifestDirectory;
+                }
+            }
+
+            throw new FileNotFoundException(string.Format(Strings.SpecifiedManifestNotFound, manifestSpecifier.ToString()));
+        }
+
         private bool IsManifestIdOutdated(string workloadManifestDir)
         {
             var manifestId = Path.GetFileName(workloadManifestDir);

src/Resolvers/Microsoft.NET.Sdk.WorkloadManifestReader/Strings.resx

@@ -189,4 +189,7 @@
   <data name="InvalidManifestVersion" xml:space="preserve">
     <value>Invalid version: {0}</value>
   </data>
+  <data name="SpecifiedManifestNotFound" xml:space="preserve">
+    <value>Specified workload manifest was not found: {0}</value>
+  </data>
 </root>

src/Resolvers/Microsoft.NET.Sdk.WorkloadManifestReader/xlf/Strings.cs.xlf

@@ -97,6 +97,11 @@
         <target state="translated">Přesměrování úlohy {0} má jiné klíče než redirect-to.</target>
         <note />
       </trans-unit>
+      <trans-unit id="SpecifiedManifestNotFound">
+        <source>Specified workload manifest was not found: {0}</source>
+        <target state="new">Specified workload manifest was not found: {0}</target>
+        <note />
+      </trans-unit>
       <trans-unit id="UnexpectedTokenAtOffset">
         <source>Unexpected token '{0}' at offset {1}</source>
         <target state="translated">Neočekávaný token {0} u posunu {1}</target>

src/Resolvers/Microsoft.NET.Sdk.WorkloadManifestReader/xlf/Strings.de.xlf

@@ -97,6 +97,11 @@
         <target state="translated">Die Umleitungsworkload „{0}“ hat andere Schlüssel als „redirect-to“.</target>
         <note />
       </trans-unit>
+      <trans-unit id="SpecifiedManifestNotFound">
+        <source>Specified workload manifest was not found: {0}</source>
+        <target state="new">Specified workload manifest was not found: {0}</target>
+        <note />
+      </trans-unit>
       <trans-unit id="UnexpectedTokenAtOffset">
         <source>Unexpected token '{0}' at offset {1}</source>
         <target state="translated">Unerwartetes Token "{0}" bei Offset {1}.</target>

src/Resolvers/Microsoft.NET.Sdk.WorkloadManifestReader/xlf/Strings.es.xlf

@@ -97,6 +97,11 @@
         <target state="translated">La carga de trabajo de redireccionamiento '{0}' tiene claves distintas que las de 'redirect-to'</target>
         <note />
       </trans-unit>
+      <trans-unit id="SpecifiedManifestNotFound">
+        <source>Specified workload manifest was not found: {0}</source>
+        <target state="new">Specified workload manifest was not found: {0}</target>
+        <note />
+      </trans-unit>
       <trans-unit id="UnexpectedTokenAtOffset">
         <source>Unexpected token '{0}' at offset {1}</source>
         <target state="translated">Token "{0}" inesperado en el desplazamiento {1}</target>

src/Resolvers/Microsoft.NET.Sdk.WorkloadManifestReader/xlf/Strings.fr.xlf

@@ -97,6 +97,11 @@
         <target state="translated">La charge de travail de redirection « {0} » a des clés autres que « redirection vers ».</target>
         <note />
       </trans-unit>
+      <trans-unit id="SpecifiedManifestNotFound">
+        <source>Specified workload manifest was not found: {0}</source>
+        <target state="new">Specified workload manifest was not found: {0}</target>
+        <note />
+      </trans-unit>
       <trans-unit id="UnexpectedTokenAtOffset">
         <source>Unexpected token '{0}' at offset {1}</source>
         <target state="translated">Jeton '{0}' inattendu à l'offset {1}</target>

src/Resolvers/Microsoft.NET.Sdk.WorkloadManifestReader/xlf/Strings.it.xlf

@@ -97,6 +97,11 @@
         <target state="translated">Il carico di lavoro '{0}' di reindirizzamento ha chiavi diverse da ' Redirect-to '</target>
         <note />
       </trans-unit>
+      <trans-unit id="SpecifiedManifestNotFound">
+        <source>Specified workload manifest was not found: {0}</source>
+        <target state="new">Specified workload manifest was not found: {0}</target>
+        <note />
+      </trans-unit>
       <trans-unit id="UnexpectedTokenAtOffset">
         <source>Unexpected token '{0}' at offset {1}</source>
         <target state="translated">Token '{0}' imprevisto alla posizione di offset {1}</target>

src/Resolvers/Microsoft.NET.Sdk.WorkloadManifestReader/xlf/Strings.ja.xlf

@@ -97,6 +97,11 @@
         <target state="translated">リダイレクト ワークロード '{0}' に 'redirect-to' 以外のキーがあります</target>
         <note />
       </trans-unit>
+      <trans-unit id="SpecifiedManifestNotFound">
+        <source>Specified workload manifest was not found: {0}</source>
+        <target state="new">Specified workload manifest was not found: {0}</target>
+        <note />
+      </trans-unit>
       <trans-unit id="UnexpectedTokenAtOffset">
         <source>Unexpected token '{0}' at offset {1}</source>
         <target state="translated">オフセット {1} に予期しないトークン '{0}' があります</target>