From e0355b97ee2eb4ee2e15df624ba36c495935e41b Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 22 Nov 2023 02:59:06 -0500 Subject: [PATCH] [wasi] Initial wasi/aot build support (#95051) * EmccCompile: Add CompilerBinaryPath input property * [wasi] Add default rsp files to the runtime pack * [wasi] Add build support for AOT * [wasi] Add simple aot build tests * [wasi] update build-tasks target * cleanup * Import monoaotcompiler pack for wasi * fix windows build * [wasi] Enable WasmDedup by default, same as wasm * [wasi] WBT: run aot tests without _wasmDevel, to avoid running out of memory when compiling with -O0 * [wasi] Fix wasi-link.rsp path on windows --- eng/liveBuilds.targets | 2 + .../Directory.Build.props | 2 + .../WorkloadManifest.targets.in | 2 +- src/mono/wasi/Makefile | 2 +- .../Wasi.Build.Tests/WasiTemplateTests.cs | 15 +- src/mono/wasi/build/WasiApp.Native.targets | 320 +++++++++--------- src/mono/wasi/build/WasiApp.targets | 2 +- src/mono/wasi/build/WasiSdk.Defaults.props | 2 + src/mono/wasi/wasi.proj | 1 + src/mono/wasm/build/WasmApp.Native.targets | 4 - src/tasks/WasmAppBuilder/EmccCompile.cs | 3 +- 11 files changed, 176 insertions(+), 179 deletions(-) diff --git a/eng/liveBuilds.targets b/eng/liveBuilds.targets index 51f990bd229ea..51f71f379d9a3 100644 --- a/eng/liveBuilds.targets +++ b/eng/liveBuilds.targets @@ -244,6 +244,8 @@ IsNative="true" /> diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.props b/src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.props index aea7e2e5d2af6..85a76d7137821 100644 --- a/src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.props +++ b/src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.props @@ -229,6 +229,8 @@ + + diff --git a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.targets.in b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.targets.in index adb4957265b35..23aa8dbb6806f 100644 --- a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.targets.in +++ b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.targets.in @@ -95,7 +95,7 @@ - + diff --git a/src/mono/wasi/Makefile b/src/mono/wasi/Makefile index 9cd33abfa2f00..f80efedad528d 100644 --- a/src/mono/wasi/Makefile +++ b/src/mono/wasi/Makefile @@ -52,7 +52,7 @@ app-builder: $(DOTNET) build $(TOP)/src/tasks/WasmAppBuilder build-tasks: - $(DOTNET) build $(TOP)/src/tasks/WasmBuildTasks $(MSBUILD_ARGS) + $(DOTNET) build $(TOP)/src/tasks/tasks.proj /p:Configuration=$(CONFIG) $(MSBUILD_ARGS) clean: $(RM) -rf $(BUILDS_OBJ_DIR) diff --git a/src/mono/wasi/Wasi.Build.Tests/WasiTemplateTests.cs b/src/mono/wasi/Wasi.Build.Tests/WasiTemplateTests.cs index dc2829e05bef5..e26b9e34ca587 100644 --- a/src/mono/wasi/Wasi.Build.Tests/WasiTemplateTests.cs +++ b/src/mono/wasi/Wasi.Build.Tests/WasiTemplateTests.cs @@ -21,18 +21,25 @@ public WasiTemplateTests(ITestOutputHelper output, SharedBuildPerTestClassFixtur } [Theory] - [InlineData("Debug")] - [InlineData("Release")] - public void ConsoleBuildThenPublish(string config) + [InlineData("Debug", /*aot*/ false)] + [InlineData("Debug", /*aot*/ true)] + [InlineData("Release", /*aot*/ false)] + [InlineData("Release", /*aot*/ true)] + public void ConsoleBuildThenPublish(string config, bool aot) { string id = $"{config}_{GetRandomId()}"; string projectFile = CreateWasmTemplateProject(id, "wasiconsole"); string projectName = Path.GetFileNameWithoutExtension(projectFile); File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_simpleMainWithArgs); - var buildArgs = new BuildArgs(projectName, config, false, id, null); + var buildArgs = new BuildArgs(projectName, config, true, id, null); buildArgs = ExpandBuildArgs(buildArgs); + if (aot) + { + AddItemsPropertiesToProject(projectFile, "true<_WasmDevel>false"); + } + BuildProject(buildArgs, id: id, new BuildProjectOptions( diff --git a/src/mono/wasi/build/WasiApp.Native.targets b/src/mono/wasi/build/WasiApp.Native.targets index dd28bd6dd34ef..21c8d607589eb 100644 --- a/src/mono/wasi/build/WasiApp.Native.targets +++ b/src/mono/wasi/build/WasiApp.Native.targets @@ -1,25 +1,29 @@ + + + <_WasiBuildNativeCoreDependsOn> - _SetupWasiSdk; - _SetWasmBuildNativeDefaults; + _WasmAotCompileApp; + _WasmStripAOTAssemblies; _PrepareForWasiBuildNative; _GetNativeFilesForLinking; _GenerateManagedToNative; + _WasmCompileNativeFiles; _GenerateAssemblyObjectFiles; - _WasiLink; + _WasiLinkDotNet; - - - - - + <_BeforeWasmBuildAppDependsOn> + $(_BeforeWasmBuildAppDependsOn); + _SetupWasiSdk; + _SetWasmBuildNativeDefaults; + <_ExeExt Condition="$([MSBuild]::IsOSPlatform('WINDOWS'))">.exe @@ -48,6 +52,10 @@ <_ToolchainMissingErrorMessage Condition="'$(_ToolchainMissingErrorMessage)' == '' and '$(_ToolchainMissingPaths)' != ''">Using WASI_SDK_PATH=$(WASI_SDK_PATH), cannot find $(_ToolchainMissingPaths) . <_IsToolchainMissing Condition="'$(_ToolchainMissingErrorMessage)' != ''">true + + + $([MSBuild]::NormalizeDirectory($(WasiSdkBinPath))) + @@ -107,8 +115,8 @@ <_MonoAotCrossCompilerPath>@(MonoAotCrossCompiler->WithMetadataValue('RuntimeIdentifier','wasi-wasm')) - <_WasiClangDefaultFlagsRsp>$([MSBuild]::NormalizePath($(_WasmRuntimePackSrcDir), 'emcc-default.rsp')) - <_WasiClangDefaultLinkFlagsRsp>$([MSBuild]::NormalizePath($(_WasmRuntimePackSrcDir), 'emcc-link.rsp')) + <_WasiClangDefaultFlagsRsp>$([MSBuild]::NormalizePath($(_WasmRuntimePackSrcDir), 'wasi-default.rsp')) + <_WasiClangDefaultLinkFlagsRsp>$([MSBuild]::NormalizePath($(_WasmRuntimePackSrcDir), 'wasi-link.rsp')) false true true @@ -128,37 +136,26 @@ <_WasmDevel Condition="'$(_WasmDevel)' == '' and '$(WasmBuildNative)' == 'true' and '$(Configuration)' == 'Debug'">true - - - + <_WasiClangOptimizationFlagDefault Condition="'$(_WasmDevel)' == 'true'">-O0 + <_WasiClangOptimizationFlagDefault Condition="'$(_WasiClangOptimizationFlagDefault)' == '' and '$(Configuration)' == 'Debug' and '$(WasmBuildingForNestedPublish)' != 'true'">-O1 + <_WasiClangOptimizationFlagDefault Condition="'$(_WasiClangOptimizationFlagDefault)' == ''">-Oz - - - + $(_WasiClangOptimizationFlagDefault) + -O2 + $(WasiClangCompileOptimizationFlag) - - - + <_WasiClangCompileRsp>$(_WasmIntermediateOutputPath)wasi-compile.rsp + <_WasiClangCompileOutputMessageImportance Condition="'$(WasiClangVerbose)' == 'true'">Normal + <_WasiClangCompileOutputMessageImportance Condition="'$(WasiClangVerbose)' != 'true'">Low - - + <_WasiClangCompileBitcodeRsp>$(_WasmIntermediateOutputPath)wasi-compile-bc.rsp + <_WasiClangLinkRsp>$(_WasmIntermediateOutputPath)clang-link.rsp - + 52428800 - <_WasmLinkDependencies Remove="@(_WasmLinkDependencies)" /> - - - - - - - - - - <_WasmCommonIncludePaths Include="$(_WasmIntermediateOutputPath.TrimEnd('\/'))" /> <_WasmCommonIncludePaths Include="$(_WasmRuntimePackIncludeDir)mono-2.0" /> <_WasmCommonIncludePaths Include="$(_WasmRuntimePackIncludeDir)wasm" /> @@ -172,56 +169,51 @@ + <_WasmLinkDependencies Remove="@(_WasmLinkDependencies)" /> + + <_WasiClangCommonFlags Include="$(_DefaultWasiClangFlags)" /> + <_WasiClangCommonFlags Include="$(WasiClangFlags)" /> + <_WasiClangCommonFlags Include="--sysroot="$(WasiSdkRoot.Replace('\', '/'))/share/wasi-sysroot"" /> + <_WasiClangCommonFlags Include="-g" Condition="'$(WasmNativeStrip)' == 'false'" /> + <_WasiClangCommonFlags Include="-v" Condition="'$(WasiClangVerbose)' != 'false'" /> + + + + <_WasmCommonCFlags Include="-DGEN_PINVOKE=1" /> + <_WasmCommonCFlags Condition="'$(RunAOTCompilation)' == 'true'" Include="-DENABLE_AOT=1" /> + <_WasmCommonCFlags Condition="'$(_DriverGenCNeeded)' == 'true'" Include="-DDRIVER_GEN=1" /> <_WasmCommonCFlags Condition="'$(WasmSingleFileBundle)' == 'true'" Include="-DWASM_SINGLE_FILE=1" /> <_WasmCommonCFlags Condition="'$(InvariantGlobalization)' == 'true'" Include="-DINVARIANT_GLOBALIZATION=1" /> <_WasmCommonCFlags Condition="'$(InvariantTimezone)' == 'true'" Include="-DINVARIANT_TIMEZONE=1" /> <_WasmCommonCFlags Condition="'$(WasmLinkIcalls)' == 'true'" Include="-DLINK_ICALLS=1" /> + <_WasiClangCFlags Include="@(_WasmCommonCFlags)" /> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + <_WasiClangCFlags Include=""-I%(_WasmCommonIncludePaths.Identity)"" /> + <_WasiClangCFlags Include="-I"$(MicrosoftNetCoreAppRuntimePackRidNativeDir.Replace('\', '/'))include"" /> - - - - - + <_WasiClangCFlags Condition="'@(WasiAfterRuntimeLoadedDeclarations)' != ''" + Include="-D WASI_AFTER_RUNTIME_LOADED_DECLARATIONS="@(WasiAfterRuntimeLoadedDeclarations, ' ')"" /> + <_WasiClangCFlags Condition="'@(WasiAfterRuntimeLoadedCalls)' != ''" + Include="-D WASI_AFTER_RUNTIME_LOADED_CALLS="@(WasiAfterRuntimeLoadedCalls, ' ')"" /> - <_WasmNativeFileForLinking Include="@(NativeFileReference)" /> + <_WasiClangLDFlags Include="$(WasiClangLinkOptimizationFlag)" /> + <_WasiClangLDFlags Include="@(_WasiClangCommonFlags)" /> + - - - - + + <_DriverCDependencies Include="$(_WasmPInvokeHPath);$(_WasmICallTablePath)" /> + <_DriverCDependencies Include="$(_DriverGenCPath)" Condition="'$(_DriverGenCNeeded)' == 'true'" /> + + <_WasmRuntimePackSrcFile Include="$(_WasmRuntimePackSrcDir)pinvoke.c" + Dependencies="$(_WasmPInvokeHPath);$(_WasmPInvokeTablePath)" /> + <_WasmRuntimePackSrcFile Include="$(_WasmRuntimePackSrcDir)driver.c" + Dependencies="@(_DriverCDependencies)" /> + <_WasmRuntimePackSrcFile Include="$(_WasmRuntimePackSrcDir)main.c" /> + <_WasmRuntimePackSrcFile Include="$(_WasmRuntimePackSrcDir)stubs.c" /> + <_WasmRuntimePackSrcFile Include="$(_WasmRuntimePackSrcDir)synthetic-pthread.c" /> + + <_WasmRuntimePackSrcFile ObjectFile="$(_WasmIntermediateOutputPath)%(FileName).o" /> @@ -264,6 +256,70 @@ + + + <_WasiClangCFlags Include="$(WasiClangExtraCFlags)" /> + + + + + + + + + <_WasmSourceFileToCompile Remove="@(_WasmSourceFileToCompile)" /> + <_WasmSourceFileToCompile Include="@(_WasmRuntimePackSrcFile)" Dependencies="%(_WasmRuntimePackSrcFile.Dependencies);$(_WasiClangDefaultFlagsRsp);$(_WasiClangCompileRsp)" /> + + + + + + + <_WasmNativeFileForLinking Include="%(_WasmSourceFileToCompile.ObjectFile)" /> + + + + + + + <_BitCodeFile Dependencies="%(_BitCodeFile.Dependencies);$(_WasiClangDefaultFlagsRsp);$(_WasiClangCompileBitcodeRsp)" /> + + + + + + + + + + <_BitcodeLDFlags Include="@(_WasiClangLDFlags)" /> + <_BitcodeLDFlags Include="$(WasiClangExtraBitcodeLDFlags)" /> + + + + + + + @@ -282,7 +338,7 @@ - + <_WasmNativeFileForLinking Include="%(_BitcodeFile.ObjectFile)" /> <_MonoRuntimeComponentDontLink Include="libmono-component-diagnostics_tracing-static.a" /> <_MonoRuntimeComponentDontLink Include="wasm-bundled-timezones.a" Condition="'$(InvariantTimezone)' == 'true'"/> @@ -300,64 +356,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -384,7 +382,7 @@ - + @@ -408,13 +406,16 @@ - + - - - + + @@ -432,51 +433,39 @@ Lines="const char* dotnet_wasi_getentrypointassemblyname() { return "managed/$(WasmMainAssemblyFileName)"%3B }" WriteOnlyWhenDifferent="true" /> - - - <_WasmRuntimePackSrcFile Include="$(MicrosoftNetCoreAppRuntimePackRidNativeDir)src/*.c" /> - + <_WasiClangXLinkerFlags Include="--initial-memory=$(WasmInitialHeapSize)" /> - - - <_WasiSdkLinkerFlags Include="--initial-memory=$(WasmInitialHeapSize)" /> + <_WasmNativeFileForLinking Include="@(NativeFileReference)" /> <_WasiFilePathForFixup Include="$(_WasiGetEntrypointCFile)" /> <_WasiFilePathForFixup Include="@(_WasiObjectFilesForBundle)" /> <_WasiFilePathForFixup Include="@(_WasmNativeFileForLinking)" /> - <_WasiFilePathForFixup Include="@(_WasmRuntimePackSrcFile)" /> <_WasiSdkClangArgs Condition="'$(OS)' == 'Windows_NT'" Include=""$([System.String]::new(%(_WasiFilePathForFixup.Identity)).Replace('\', '/'))"" /> <_WasiSdkClangArgs Condition="'$(OS)' != 'Windows_NT'" Include="@(_WasiFilePathForFixup -> '"%(Identity)"')" /> - <_WasiSdkClangArgs Include="@(_WasmCommonCFlags)" /> - - <_WasiSdkClangArgs Include=""-I%(_WasmCommonIncludePaths.Identity)"" /> - <_WasiSdkClangArgs Include="--sysroot="$(WasiSdkRoot.Replace('\', '/'))/share/wasi-sysroot"" /> - <_WasiSdkClangArgs Include="-I"$(MicrosoftNetCoreAppRuntimePackRidNativeDir.Replace('\', '/'))include"" /> <_WasiSdkClangArgs Include="-Wl,--export=malloc,--export=free,--export=__heap_base,--export=__data_end" /> <_WasiSdkClangArgs Include="-Wl,-z,stack-size=8388608,-lwasi-emulated-process-clocks,-lwasi-emulated-signal,-lwasi-emulated-mman"/> <_WasiSdkClangArgs Include="-Wl,-s" Condition="'$(WasmNativeStrip)' == 'true'"/> - <_WasiSdkClangArgs Include="@(_WasiSdkLinkerFlags -> '-Xlinker %(Identity)', ' ')" /> + <_WasiSdkClangArgs Include=""@$(_WasiClangDefaultLinkFlagsRsp.Replace('\', '/'))"" /> + <_WasiSdkClangArgs Include="@(_WasiClangXLinkerFlags -> '-Xlinker %(Identity)', ' ')" /> + <_WasiSdkClangArgs Include="@(_WasiClangLDFlags)" /> - <_WasiSdkClangArgs Condition="'@(WasiAfterRuntimeLoadedDeclarations)' != ''" - Include="-D WASI_AFTER_RUNTIME_LOADED_DECLARATIONS="@(WasiAfterRuntimeLoadedDeclarations, ' ')"" /> - <_WasiSdkClangArgs Condition="'@(WasiAfterRuntimeLoadedCalls)' != ''" - Include="-D WASI_AFTER_RUNTIME_LOADED_CALLS="@(WasiAfterRuntimeLoadedCalls, ' ')"" /> <_WasiSdkClangArgs Include="-o "$(_WasmOutputFileName.Replace('\', '/'))"" /> - + + - + - + @@ -488,15 +477,9 @@ - - - - - - - <_MonoAotCrossCompilerPath>@(MonoAotCrossCompiler->WithMetadataValue('RuntimeIdentifier','browser-wasm')) + <_MonoAotCrossCompilerPath>@(MonoAotCrossCompiler->WithMetadataValue('RuntimeIdentifier', 'wasi-wasm')) @@ -589,7 +572,7 @@ DedupAssembly="$(_WasmDedupAssembly)" CacheFilePath="$(_AOTCompilerCacheFile)" LLVMDebug="dwarfdebug" - LLVMPath="$(EmscriptenUpstreamBinPath)" + LLVMPath="$(WasiSdkBinPath)" IntermediateOutputPath="$(_WasmIntermediateOutputPath)" AotProfilePath="@(AotProfilePath)"> @@ -629,4 +612,7 @@ <_WasmAssembliesInternal Include="@(_WasmStrippedAssemblies)" /> + + + diff --git a/src/mono/wasi/build/WasiApp.targets b/src/mono/wasi/build/WasiApp.targets index 1811e61146446..975411c690c20 100644 --- a/src/mono/wasi/build/WasiApp.targets +++ b/src/mono/wasi/build/WasiApp.targets @@ -77,7 +77,7 @@ --> - false + true false false diff --git a/src/mono/wasi/build/WasiSdk.Defaults.props b/src/mono/wasi/build/WasiSdk.Defaults.props index baa50e602670f..ab6bd76a4c748 100644 --- a/src/mono/wasi/build/WasiSdk.Defaults.props +++ b/src/mono/wasi/build/WasiSdk.Defaults.props @@ -4,5 +4,7 @@ $([MSBuild]::NormalizeDirectory($(WasiSdkRoot), 'share', 'wasi-sysroot')) $(WasiSdkRoot)\bin\clang $(WasiClang).exe + + $([MSBuild]::NormalizeDirectory($(WasiSdkRoot), 'bin')) diff --git a/src/mono/wasi/wasi.proj b/src/mono/wasi/wasi.proj index b942b6f492503..9905c32db73f5 100644 --- a/src/mono/wasi/wasi.proj +++ b/src/mono/wasi/wasi.proj @@ -264,6 +264,7 @@ + diff --git a/src/mono/wasm/build/WasmApp.Native.targets b/src/mono/wasm/build/WasmApp.Native.targets index 6a236779d3cf7..438ea50f3fd17 100644 --- a/src/mono/wasm/build/WasmApp.Native.targets +++ b/src/mono/wasm/build/WasmApp.Native.targets @@ -11,7 +11,6 @@ _WasmAotCompileApp; _WasmStripAOTAssemblies; _PrepareForWasmBuildNative; - _GenerateDriverGenC; _GenerateManagedToNative; _WasmCompileNativeFiles; _WasmLinkDotNet; @@ -549,9 +548,6 @@ - - - diff --git a/src/tasks/WasmAppBuilder/EmccCompile.cs b/src/tasks/WasmAppBuilder/EmccCompile.cs index eb157e8ec39a7..0373abea393a8 100644 --- a/src/tasks/WasmAppBuilder/EmccCompile.cs +++ b/src/tasks/WasmAppBuilder/EmccCompile.cs @@ -34,6 +34,7 @@ public class EmccCompile : Microsoft.Build.Utilities.Task public string? WorkingDirectory { get; set; } public string OutputMessageImportance{ get; set; } = "Low"; public string? MessageToIndicateCompiling { get; set; } + public string CompilerBinaryPath { get; set; } = "emcc"; [Output] public ITaskItem[]? OutputFiles { get; private set; } @@ -192,7 +193,7 @@ bool ProcessSourceFile(string srcFile, string objFile) string tmpObjFile = Path.GetTempFileName(); try { - string command = $"emcc {Arguments} -c -o \"{tmpObjFile}\" \"{srcFile}\""; + string command = $"{CompilerBinaryPath} {Arguments} -c -o \"{tmpObjFile}\" \"{srcFile}\""; var startTime = DateTime.Now; // Log the command in a compact format which can be copy pasted