[browser] Use StaticWebAssets fingerprinting in Wasm SDK (#103755) (#…

…105294)

* Build and publish integration

* Make fingerpring work at runtime for assemblies

* Make fingerpring work at runtime for icu

* Remove version fingerprint check

* Check core assembly extension

* Typescript nits

* JSModules and SatelliteAssemblies

* DEBUG require newer SDK for testing

* Fix fingerprint for new publish assets

* Lazy loading and FP mapping boot json

* WBT file on disk checks

* WBT file on disk checks

* WBT file on disk checks

* WBT testmain no fingerprint

* WBT revert debug message

* AOT

* WBT fix ordering

* Fingerprinting without webcil

* Fix GenerateWasmBootJson when FP is off

* NoFingerprint WBT variant

* DEBUG try to run WBT without fingerprinting

* WBT make entry comparison order agnostic

* WBT smoke tests for no-fingerprinting

* Update sendtohelix-browser.targets

* Remove debug log

* Fix typo

* Fix regex matching

* Remove test for dotnet.js FP since we don't support that anymore

* Fix check for System.Private.CoreLib

* FP for dotnet.globalization.js

* Fingerprinting pdbs

* WBT fix file check

* Fingerprint segmentation-rules.json

* Fix loading pdb for fingerprinted lazy assembly

* Ensure lazy pdb is loaded

* Remove non-WasmSDK tests from non-FP category

* Revert drop for dotnet.js finterprinting

* Compute non-Fingerprinted virtualPath for pdb and resource as well

* Make debugger working with fingerprinted assemblies and pdbs

* DEBUG latest SDK for WBT

* DEBUG fix wbt installation

* Add WorkloadBuildTasks to WasmBuild.sln

* Fix WBT

* Revert escaping URL in debugger

* Fix lazy loading test and message emit in release config

* Fixes for MT after merge

* Skip WBT without workloads and without fingerprinting

* Turn off fingerprinting when targeting downlevel versions

* Git ignore *.d.ts.sha256

* Fix

* Update source-build-reference-packages to latest

* Revert "Update source-build-reference-packages to latest"

This reverts commit bef50ee.

* Fix the references

* Update Versions.props

* Update Versions.props

---------

Co-authored-by: Larry Ewing <lewing@microsoft.com>

src/libraries/sendtohelix-browser.targets

@@ -139,6 +139,7 @@
     <BuildWasmAppsJobsList>$(RepositoryEngineeringDir)testing\scenarios\BuildWasmAppsJobsList.txt</BuildWasmAppsJobsList>
     <_XUnitTraitArg Condition="'$(TestUsingWorkloads)' == 'true'">-notrait category=no-workload</_XUnitTraitArg>
     <_XUnitTraitArg Condition="'$(TestUsingWorkloads)' != 'true'">-trait category=no-workload</_XUnitTraitArg>
+    <_XUnitTraitArg Condition="'$(TestUsingFingerprinting)' == 'false'">$(_XUnitTraitArg) -trait category=no-fingerprinting</_XUnitTraitArg>
   </PropertyGroup>
 
   <PropertyGroup>

src/libraries/sendtohelix-wasm.targets

@@ -13,6 +13,7 @@
     <WorkItemPrefix Condition="'$(TestUsingWorkloads)' == 'true'">Workloads-</WorkItemPrefix>
     <WorkItemPrefix Condition="'$(TestUsingWorkloads)' != 'true'">NoWorkload-</WorkItemPrefix>
     <WorkItemPrefix Condition="'$(TestUsingWebcil)' == 'false'">$(WorkItemPrefix)NoWebcil-</WorkItemPrefix>
+    <WorkItemPrefix Condition="'$(TestUsingFingerprinting)' == 'false'">$(WorkItemPrefix)NoFingerprint-</WorkItemPrefix>
     <WorkItemPrefix Condition="'$(WasmEnableThreads)' != 'true'">$(WorkItemPrefix)ST-</WorkItemPrefix>
     <WorkItemPrefix Condition="'$(WasmEnableThreads)' == 'true'">$(WorkItemPrefix)MT-</WorkItemPrefix>
   </PropertyGroup>
@@ -49,15 +50,15 @@
 
     <!-- for testing with workloads, we use separate items -->
     <ItemGroup>
-      <HelixWorkItem Include="@(BuildWasmApps_PerJobList->'$(WorkItemPrefix)%(Identity)')" Condition="'$(TestUsingWorkloads)' == 'true'">
+      <HelixWorkItem Include="@(BuildWasmApps_PerJobList->'$(WorkItemPrefix)%(Identity)')" Condition="'$(TestUsingWorkloads)' == 'true' and '$(TestUsingFingerprinting)' == 'true'">
         <PayloadArchive>$(_BuildWasmAppsPayloadArchive)</PayloadArchive>
         <PreCommands Condition="'$(OS)' == 'Windows_NT'">set &quot;HELIX_XUNIT_ARGS=-class %(Identity)&quot;</PreCommands>
         <PreCommands Condition="'$(OS)' != 'Windows_NT'">export &quot;HELIX_XUNIT_ARGS=-class %(Identity)&quot;</PreCommands>
         <Command>$(HelixCommand)</Command>
         <Timeout>$(_workItemTimeout)</Timeout>
       </HelixWorkItem>
 
-      <HelixWorkItem Include="$(WorkItemPrefix)Wasm.Build.Tests" Condition="'$(TestUsingWorkloads)' != 'true'">
+      <HelixWorkItem Include="$(WorkItemPrefix)Wasm.Build.Tests" Condition="'$(TestUsingWorkloads)' != 'true' or '$(TestUsingFingerprinting)' != 'true'">
         <PayloadArchive>$(_BuildWasmAppsPayloadArchive)</PayloadArchive>
         <Command>$(HelixCommand)</Command>
         <Timeout>$(_workItemTimeout)</Timeout>

src/libraries/sendtohelix.proj

@@ -87,17 +87,24 @@
     <ItemGroup Condition="'@(_Scenarios -> AnyHaveMetadataValue('Identity', 'buildwasmapps'))' == 'true'">
       <_TestUsingWorkloadsValues Include="true;false" />
       <_TestUsingWebcilValues Include="true;false" Condition="'$(TargetOS)' == 'browser'" />
+      <_TestUsingFingerprintingValues Include="true;false" Condition="'$(TargetOS)' == 'browser'" />
 
       <!-- now make the cartesian product of true and false values for two categories -->
       <_TestUsingCrossProductValuesTemp Include="@(_TestUsingWorkloadsValues)">
         <Workloads>%(_TestUsingWorkloadsValues.Identity)</Workloads>
       </_TestUsingCrossProductValuesTemp>
-      <_TestUsingCrossProductValues Include="@(_TestUsingCrossProductValuesTemp)">
+      <_TestUsingCrossProductValuesTemp2 Include="@(_TestUsingCrossProductValuesTemp)">
         <Webcil>%(_TestUsingWebcilValues.Identity)</Webcil>
+      </_TestUsingCrossProductValuesTemp2>
+      <_TestUsingCrossProductValues Include="@(_TestUsingCrossProductValuesTemp2)">
+        <Fingerprinting>%(_TestUsingFingerprintingValues.Identity)</Fingerprinting>
       </_TestUsingCrossProductValues>
 
+      <!-- There no tests without fingerprinting and without workloads -->
+      <_TestUsingCrossProductValues Remove="@(_TestUsingCrossProductValues)" Condition="'%(_TestUsingCrossProductValues.Workloads)' == 'false' and '%(_TestUsingCrossProductValues.Fingerprinting)' == 'false'" />
+
       <_BuildWasmAppsProjectsToBuild Include="$(PerScenarioProjectFile)">
-        <AdditionalProperties>$(_PropertiesToPass);Scenario=BuildWasmApps;TestArchiveRuntimeFile=$(TestArchiveRuntimeFile);TestUsingWorkloads=%(_TestUsingCrossProductValues.Workloads);TestUsingWebcil=%(_TestUsingCrossProductValues.Webcil)</AdditionalProperties>
+        <AdditionalProperties>$(_PropertiesToPass);Scenario=BuildWasmApps;TestArchiveRuntimeFile=$(TestArchiveRuntimeFile);TestUsingWorkloads=%(_TestUsingCrossProductValues.Workloads);TestUsingWebcil=%(_TestUsingCrossProductValues.Webcil);TestUsingFingerprinting=%(_TestUsingCrossProductValues.Fingerprinting)</AdditionalProperties>
         <AdditionalProperties Condition="'$(NeedsToBuildWasmAppsOnHelix)' != ''">%(_BuildWasmAppsProjectsToBuild.AdditionalProperties);NeedsToBuildWasmAppsOnHelix=$(NeedsToBuildWasmAppsOnHelix)</AdditionalProperties>
       </_BuildWasmAppsProjectsToBuild>
     </ItemGroup>

src/libraries/sendtohelixhelp.proj

@@ -157,6 +157,7 @@
     <HelixCommandPrefixEnvVarItem Include="DOTNET_CLI_TELEMETRY_OPTOUT=1" />
     <HelixCommandPrefixEnvVarItem Condition="'$(TestUsingWorkloads)' == 'true'" Include="TEST_USING_WORKLOADS=true" />
     <HelixCommandPrefixEnvVarItem Condition="'$(TestUsingWebcil)' == 'false'" Include="TEST_USING_WEBCIL=false" />
+    <HelixCommandPrefixEnvVarItem Condition="'$(TestUsingFingerprinting)' == 'false'" Include="TEST_USING_FINGERPRINTING=false" />
     <HelixCommandPrefixEnvVarItem Condition="'$(WorkloadsTestPreviousVersions)' == 'true'" Include="WORKLOADS_TEST_PREVIOUS_VERSIONS=true" />
   </ItemGroup>
 
@@ -346,7 +347,7 @@
   <Target Name="PrintHelixQueues">
     <Message Importance="High" Text="Using Queues: $(HelixTargetQueues)" />
     <Message Condition="'$(Scenario)' == 'BuildWasmApps'" Importance="High"
-             Text="Scenario: $(Scenario), TestUsingWorkloads: $(TestUsingWorkloads), TestUsingWebcil: $(TestUsingWebcil)" />
+             Text="Scenario: $(Scenario), TestUsingWorkloads: $(TestUsingWorkloads), TestUsingWebcil: $(TestUsingWebcil), TestUsingFingerprinting: $(TestUsingFingerprinting)" />
   </Target>
 
   <Target Name="PrintBuildTargetFramework">

src/mono/browser/.gitignore

@@ -2,4 +2,4 @@
 .stamp-wasm-install-and-select*
 emsdk
 
-runtime/dotnet.d.ts.sha256
+runtime/*.d.ts.sha256

src/mono/browser/debugger/BrowserDebugProxy/DebugStore.cs

@@ -1683,6 +1683,9 @@ public async IAsyncEnumerable<SourceFile> Load(SessionId id, string[] loaded_fil
             var asm_files = new List<string>();
             List<DebugItem> steps = new List<DebugItem>();
 
+            // Use System.Private.CoreLib to determine if we have a fingerprinted assemblies or not.
+            bool isFingerprinted = Path.GetFileNameWithoutExtension(loaded_files.FirstOrDefault(f => f.Contains("System.Private.CoreLib"))) != "System.Private.CoreLib";
+
             if (!useDebuggerProtocol)
             {
                 var pdb_files = new List<string>();
@@ -1698,8 +1701,17 @@ public async IAsyncEnumerable<SourceFile> Load(SessionId id, string[] loaded_fil
                 {
                     try
                     {
-                        string candidate_pdb = Path.ChangeExtension(url, "pdb");
-                        string pdb = pdb_files.FirstOrDefault(n => n == candidate_pdb);
+                        string pdb;
+                        if (isFingerprinted)
+                        {
+                            string noFingerprintPdbFileName = string.Concat(Path.GetFileNameWithoutExtension(Path.GetFileNameWithoutExtension(url)), ".pdb");
+                            pdb = pdb_files.FirstOrDefault(n => string.Concat(Path.GetFileNameWithoutExtension(Path.GetFileNameWithoutExtension(n)), Path.GetExtension(n)) == noFingerprintPdbFileName);
+                        }
+                        else
+                        {
+                            string candidate_pdb = Path.ChangeExtension(url, "pdb");
+                            pdb = pdb_files.FirstOrDefault(n => n == candidate_pdb);
+                        }
 
                         steps.Add(
                             new DebugItem
@@ -1722,12 +1734,14 @@ public async IAsyncEnumerable<SourceFile> Load(SessionId id, string[] loaded_fil
                         continue;
                     try
                     {
-                        string unescapedFileName = Uri.UnescapeDataString(file_name);
+                        string unescapedFileName = Path.GetFileName(Uri.UnescapeDataString(file_name));
+                        if (isFingerprinted)
+                            unescapedFileName = string.Concat(Path.GetFileNameWithoutExtension(Path.GetFileNameWithoutExtension(unescapedFileName)), Path.GetExtension(unescapedFileName));
                         steps.Add(
                             new DebugItem
                             {
                                 Url = file_name,
-                                DataTask = context.SdbAgent.GetDataFromAssemblyAndPdbAsync(Path.GetFileName(unescapedFileName), false, token)
+                                DataTask = context.SdbAgent.GetDataFromAssemblyAndPdbAsync(unescapedFileName, false, token)
                             });
                     }
                     catch (Exception e)

src/mono/browser/runtime/diagnostics-mock.d.ts

@@ -71,4 +71,4 @@ interface MockEnvironment {
     expectAdvertise: FilterPredicate;
 }
 
-export { MockEnvironment, MockScriptConnection, PromiseAndController };
+export type { MockEnvironment, MockScriptConnection, PromiseAndController };

src/mono/browser/runtime/dotnet.d.ts

@@ -247,6 +247,9 @@ type ResourceExtensions = {
 };
 interface ResourceGroups {
     hash?: string;
+    fingerprinting?: {
+        [name: string]: string;
+    };
     coreAssembly?: ResourceList;
     assembly?: ResourceList;
     lazyAssembly?: ResourceList;
@@ -692,4 +695,4 @@ declare global {
 }
 declare const createDotnetRuntime: CreateDotnetRuntimeType;
 
-export { AssetBehaviors, AssetEntry, CreateDotnetRuntimeType, DotnetHostBuilder, DotnetModuleConfig, EmscriptenModule, GlobalizationMode, IMemoryView, ModuleAPI, MonoConfig, RuntimeAPI, createDotnetRuntime as default, dotnet, exit };
+export { type AssetBehaviors, type AssetEntry, type CreateDotnetRuntimeType, type DotnetHostBuilder, type DotnetModuleConfig, type EmscriptenModule, GlobalizationMode, type IMemoryView, type ModuleAPI, type MonoConfig, type RuntimeAPI, createDotnetRuntime as default, dotnet, exit };

src/mono/browser/runtime/lazyLoading.ts

@@ -7,11 +7,23 @@ import { AssetEntry } from "./types";
 
 export async function loadLazyAssembly (assemblyNameToLoad: string): Promise<boolean> {
     const resources = loaderHelpers.config.resources!;
+    const originalAssemblyName = assemblyNameToLoad;
     const lazyAssemblies = resources.lazyAssembly;
     if (!lazyAssemblies) {
         throw new Error("No assemblies have been marked as lazy-loadable. Use the 'BlazorWebAssemblyLazyLoad' item group in your project file to enable lazy loading an assembly.");
     }
 
+    if (loaderHelpers.config.resources!.fingerprinting) {
+        const map = loaderHelpers.config.resources!.fingerprinting;
+        for (const fingerprintedName in map) {
+            const nonFingerprintedName = map[fingerprintedName];
+            if (nonFingerprintedName == assemblyNameToLoad) {
+                assemblyNameToLoad = fingerprintedName;
+                break;
+            }
+        }
+    }
+
     if (!lazyAssemblies[assemblyNameToLoad]) {
         throw new Error(`${assemblyNameToLoad} must be marked with 'BlazorWebAssemblyLazyLoad' item group in your project file to allow lazy-loading.`);
     }
@@ -26,8 +38,22 @@ export async function loadLazyAssembly (assemblyNameToLoad: string): Promise<boo
         return false;
     }
 
-    const pdbNameToLoad = changeExtension(dllAsset.name, ".pdb");
-    const shouldLoadPdb = loaderHelpers.config.debugLevel != 0 && loaderHelpers.isDebuggingSupported() && Object.prototype.hasOwnProperty.call(lazyAssemblies, pdbNameToLoad);
+    let pdbNameToLoad = changeExtension(originalAssemblyName, ".pdb");
+    let shouldLoadPdb = false;
+    if (loaderHelpers.config.debugLevel != 0 && loaderHelpers.isDebuggingSupported()) {
+        shouldLoadPdb = Object.prototype.hasOwnProperty.call(lazyAssemblies, pdbNameToLoad);
+        if (loaderHelpers.config.resources!.fingerprinting) {
+            const map = loaderHelpers.config.resources!.fingerprinting;
+            for (const fingerprintedName in map) {
+                const nonFingerprintedName = map[fingerprintedName];
+                if (nonFingerprintedName == pdbNameToLoad) {
+                    pdbNameToLoad = fingerprintedName;
+                    shouldLoadPdb = true;
+                    break;
+                }
+            }
+        }
+    }
 
     const dllBytesPromise = loaderHelpers.retrieve_asset_download(dllAsset);
 

src/mono/browser/runtime/loader/assets.ts

@@ -318,6 +318,9 @@ export function prepareAssets () {
         }
 
         const addAsset = (asset: AssetEntryInternal, isCore: boolean) => {
+            if (resources.fingerprinting && (asset.behavior == "assembly" || asset.behavior == "pdb" || asset.behavior == "resource")) {
+                asset.virtualPath = getNonFingerprintedAssetName(asset.name);
+            }
             if (isCore) {
                 asset.isCore = true;
                 coreAssetsToLoad.push(asset);
@@ -418,7 +421,7 @@ export function prepareAssets () {
                         behavior: "icu",
                         loadRemote: true
                     });
-                } else if (name === "segmentation-rules.json") {
+                } else if (name.startsWith("segmentation-rules") && name.endsWith(".json")) {
                     assetsToLoad.push({
                         name,
                         hash: resources.icu[name],
@@ -460,6 +463,15 @@ export function prepareAssets () {
     config.assets = [...coreAssetsToLoad, ...assetsToLoad, ...modulesAssets];
 }
 
+export function getNonFingerprintedAssetName (assetName: string) {
+    const fingerprinting = loaderHelpers.config.resources?.fingerprinting;
+    if (fingerprinting && fingerprinting[assetName]) {
+        return fingerprinting[assetName];
+    }
+
+    return assetName;
+}
+
 export function prepareAssetsWorker () {
     const config = loaderHelpers.config;
     mono_assert(config.assets, "config.assets must be defined");

src/mono/browser/runtime/loader/icu.ts

@@ -5,6 +5,7 @@ import { mono_log_error } from "./logging";
 import { GlobalizationMode, MonoConfig } from "../types";
 import { ENVIRONMENT_IS_WEB, loaderHelpers } from "./globals";
 import { mono_log_info, mono_log_debug } from "./logging";
+import { getNonFingerprintedAssetName } from "./assets";
 
 export function init_globalization () {
     loaderHelpers.preferredIcuAsset = getIcuResourceName(loaderHelpers.config);
@@ -51,6 +52,17 @@ export function getIcuResourceName (config: MonoConfig): string | null {
         const culture = config.applicationCulture || (ENVIRONMENT_IS_WEB ? (globalThis.navigator && globalThis.navigator.languages && globalThis.navigator.languages[0]) : Intl.DateTimeFormat().resolvedOptions().locale);
 
         const icuFiles = Object.keys(config.resources.icu);
+        const fileMapping: {
+            [k: string]: string
+        } = {};
+        for (let index = 0; index < icuFiles.length; index++) {
+            const icuFile = icuFiles[index];
+            if (config.resources.fingerprinting) {
+                fileMapping[getNonFingerprintedAssetName(icuFile)] = icuFile;
+            } else {
+                fileMapping[icuFile] = icuFile;
+            }
+        }
 
         let icuFile = null;
         if (config.globalizationMode === GlobalizationMode.Custom) {
@@ -65,8 +77,8 @@ export function getIcuResourceName (config: MonoConfig): string | null {
             icuFile = getShardedIcuResourceName(culture);
         }
 
-        if (icuFile && icuFiles.includes(icuFile)) {
-            return icuFile;
+        if (icuFile && fileMapping[icuFile]) {
+            return fileMapping[icuFile];
         }
     }
 

src/mono/browser/runtime/types/index.ts

@@ -203,6 +203,7 @@ export type ResourceExtensions = { [extensionName: string]: ResourceList };
 
 export interface ResourceGroups {
     hash?: string;
+    fingerprinting?: { [name: string]: string },
     coreAssembly?: ResourceList; // nullable only temporarily
     assembly?: ResourceList; // nullable only temporarily
     lazyAssembly?: ResourceList; // nullable only temporarily

src/mono/nuget/Microsoft.NET.Sdk.WebAssembly.Pack/build/Microsoft.NET.Sdk.WebAssembly.Browser.props

@@ -35,6 +35,7 @@ Copyright (c) .NET Foundation. All rights reserved.
     <StaticWebAssetsGetPublishAssetsTargets>ComputeFilesToPublish;GetCurrentProjectPublishStaticWebAssetItems</StaticWebAssetsGetPublishAssetsTargets>
     <StaticWebAssetsAdditionalPublishProperties>$(StaticWebAssetsAdditionalPublishProperties);BuildProjectReferences=false;ResolveAssemblyReferencesFindRelatedSatellites=true</StaticWebAssetsAdditionalPublishProperties>
     <StaticWebAssetsAdditionalPublishPropertiesToRemove>$(StaticWebAssetsAdditionalPublishPropertiesToRemove);NoBuild;RuntimeIdentifier</StaticWebAssetsAdditionalPublishPropertiesToRemove>
+    <StaticWebAssetStandaloneHosting Condition="'$(StaticWebAssetStandaloneHosting)' == '' and '$(StaticWebAssetProjectMode)' == 'Root'">true</StaticWebAssetStandaloneHosting>
   </PropertyGroup>
 
   <ItemGroup>