From c57cba795a0e0a2905f6ce48009cde1d23cd7a19 Mon Sep 17 00:00:00 2001 From: Santiago Fernandez Madero Date: Fri, 25 Oct 2019 14:19:34 -0500 Subject: [PATCH] Add support to produce WebAssembly runtime assets (#41777) * Add support to produce WebAssembly runtime assets * Add support to build in OSX and address PR feedback * Change AND -> and in Directory.Build.props change region * PR Feedback: run build in rolling CI builds as well * Add TODO comment for WebAssembly OSGroup --- .azure-ci.yml | 3 + Directory.Build.props | 22 ++++-- eng/configurations/osgroups.props | 6 +- eng/pipelines/linux.yml | 70 ++++++++++++------- eng/restore/runtime/runtime.depproj | 2 + ...osoft.XmlSerializer.Generator.Tests.csproj | 2 +- src/Native/Unix/CMakeLists.txt | 14 +++- src/Native/Unix/Common/pal_config.h.in | 1 + src/Native/Unix/System.Native/CMakeLists.txt | 36 +++++----- .../Unix/System.Native/pal_networking.c | 7 +- src/Native/Unix/System.Native/pal_process.c | 2 +- src/Native/Unix/configure.cmake | 4 ++ src/Native/Unix/gen-buildsys-clang.sh | 11 ++- src/Native/Windows/gen-buildsys-win.bat | 8 +++ src/Native/build-native.cmd | 4 -- src/Native/build-native.proj | 2 +- src/Native/build-native.sh | 47 ++++++++----- src/Native/native-binplace.proj | 1 + .../src/System.Data.Odbc.csproj | 2 +- 19 files changed, 161 insertions(+), 83 deletions(-) diff --git a/.azure-ci.yml b/.azure-ci.yml index 79d370500b275..cbbe053e47889 100644 --- a/.azure-ci.yml +++ b/.azure-ci.yml @@ -55,6 +55,9 @@ resources: - container: ubuntu_1604_arm_cross_container image: microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-ef0ac75-20175511035548 + - container: ubuntu_1604 + image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-bd0fe7c-20190923200211 + variables: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notIn(variables['Build.Reason'], 'PullRequest')) }}: - group: DotNet-Blob-Feed diff --git a/Directory.Build.props b/Directory.Build.props index de85bb3391d17..5320957e76bfe 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -57,15 +57,20 @@ $(DefaultOSGroup) Debug $([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant) - arm - arm64 + arm + arm64 + wasm x64 + + $(OSGroup.ToLowerInvariant()) + $(TargetGroup)-$(OSGroup)-$(ConfigurationGroup)-$(ArchGroup) - package + package @@ -130,13 +135,16 @@ $(_runtimeOS)-x64 $(_runtimeOS)-$(HostArch) - linux-x64 + linux-x64 - - win-x64 + + <_buildingInOSX>$([MSBuild]::IsOSPlatform('OSX')) + win-x64 + osx-x64 + linux-x64 - <_portableOS Condition="'$(OSGroup)' == 'Unix' AND '$(_runtimeOSFamily)' != 'osx' AND '$(_runtimeOSFamily)' != 'FreeBSD' AND '$(_runtimeOS)' != 'linux-musl'">linux + <_portableOS Condition="'$(OSGroup)' == 'Unix' and '$(_runtimeOSFamily)' != 'osx' and '$(_runtimeOSFamily)' != 'FreeBSD' AND '$(_runtimeOS)' != 'linux-musl'">linux <_packageRID /> <_packageRID Condition="'$(PortableBuild)' == 'true'">$(_portableOS)-$(ArchGroup) diff --git a/eng/configurations/osgroups.props b/eng/configurations/osgroups.props index 7f2a49cdd024e..f2861d4755b42 100644 --- a/eng/configurations/osgroups.props +++ b/eng/configurations/osgroups.props @@ -35,8 +35,10 @@ netbsd - Unix - true + + Linux + true true diff --git a/eng/pipelines/linux.yml b/eng/pipelines/linux.yml index a828ba28df398..c74db96752365 100644 --- a/eng/pipelines/linux.yml +++ b/eng/pipelines/linux.yml @@ -134,35 +134,57 @@ stages: - linuxArm64Queues: \(Ubuntu.1604.Arm64.Open\)Ubuntu.1604.Arm64.Docker.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-helix-arm64v8-a45aeeb-20190620160300 # Legs without helix testing - # Only run this leg in PRs. - - ${{ if and(eq(parameters.fullMatrix, 'false'), and(ne(parameters.testScope, 'outerloop'), ne(parameters.testScope, 'all'))) }}: + # Don't run these legs in outerloop builds + - ${{ if and(ne(parameters.testScope, 'outerloop'), ne(parameters.testScope, 'all')) }}: - job: LinuxNoTest displayName: Build strategy: matrix: - arm_Debug: - _BuildConfig: Debug - _architecture: arm - _framework: netcoreapp - _buildExtraArguments: /p:RuntimeOS=ubuntu.16.04 -warnAsError false - _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm ' - _dockerContainer: ubuntu_1604_arm_cross_container - - musl_arm64_Debug: - _BuildConfig: Debug - _architecture: arm64 - _framework: netcoreapp - _dockerContainer: alpine_37_arm64_container - _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm64 ' - _buildExtraArguments: -warnAsError false /p:BuildNativeCompiler=--clang5.0 /p:RuntimeOS=linux-musl - - arm64_Debug: - _BuildConfig: Debug - _architecture: arm64 + wasm_Release: + _BuildConfig: Release + _architecture: wasm _framework: netcoreapp - _dockerContainer: ubuntu_1604_arm64_cross_container - _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm64 ' - _buildExtraArguments: --warnAsError false + _dockerContainer: ubuntu_1604 + _emsdkPath: $(Build.BinariesDirectory)/emsdk + _buildScriptPrefix: 'EMSDK_PATH=$(_emsdkPath) ' + _buildExtraArguments: -os WebAssembly -warnAsError false + _installEmscripten: true + + # Only run these legs in PRs. + ${{ if eq(parameters.fullMatrix, 'false') }}: + arm_Debug: + _BuildConfig: Debug + _architecture: arm + _framework: netcoreapp + _buildExtraArguments: /p:RuntimeOS=ubuntu.16.04 -warnAsError false + _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm ' + _dockerContainer: ubuntu_1604_arm_cross_container + + musl_arm64_Debug: + _BuildConfig: Debug + _architecture: arm64 + _framework: netcoreapp + _dockerContainer: alpine_37_arm64_container + _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm64 ' + _buildExtraArguments: -warnAsError false /p:BuildNativeCompiler=--clang5.0 /p:RuntimeOS=linux-musl + + arm64_Debug: + _BuildConfig: Debug + _architecture: arm64 + _framework: netcoreapp + _dockerContainer: ubuntu_1604_arm64_cross_container + _buildScriptPrefix: 'ROOTFS_DIR=/crossrootfs/arm64 ' + _buildExtraArguments: --warnAsError false + + preBuildSteps: + - script: | + EMSCRIPTEN_VERSION=1.38.47 + git clone https://github.com/emscripten-core/emsdk.git $(_emsdkPath) + cd $(_emsdkPath) + ./emsdk install ${EMSCRIPTEN_VERSION}-upstream + ./emsdk activate --embedded ${EMSCRIPTEN_VERSION}-upstream + displayName: Install Emscripten + condition: and(succeeded(), eq(variables['_installEmscripten'], 'true')) pool: name: Hosted Ubuntu 1604 diff --git a/eng/restore/runtime/runtime.depproj b/eng/restore/runtime/runtime.depproj index 89ef92cace120..421434766c7a5 100644 --- a/eng/restore/runtime/runtime.depproj +++ b/eng/restore/runtime/runtime.depproj @@ -1,6 +1,8 @@  $(PackageRID) + + $(ToolRuntimeRID) $(NoWarn);NU1603;NU1605 true diff --git a/src/Microsoft.XmlSerializer.Generator/tests/Microsoft.XmlSerializer.Generator.Tests.csproj b/src/Microsoft.XmlSerializer.Generator/tests/Microsoft.XmlSerializer.Generator.Tests.csproj index c1844bd3ceb05..75de34b20cf96 100644 --- a/src/Microsoft.XmlSerializer.Generator/tests/Microsoft.XmlSerializer.Generator.Tests.csproj +++ b/src/Microsoft.XmlSerializer.Generator/tests/Microsoft.XmlSerializer.Generator.Tests.csproj @@ -3,7 +3,7 @@ $(DefineConstants);XMLSERIALIZERGENERATORTESTS netcoreapp-Debug;netcoreapp-Release false - true + true diff --git a/src/Native/Unix/CMakeLists.txt b/src/Native/Unix/CMakeLists.txt index f1d6a9ac599bf..e7447cf757694 100644 --- a/src/Native/Unix/CMakeLists.txt +++ b/src/Native/Unix/CMakeLists.txt @@ -18,7 +18,6 @@ add_compile_options(-Wno-empty-translation-unit) add_compile_options(-Wno-cast-align) add_compile_options(-Wno-typedef-redefinition) add_compile_options(-Wno-c11-extensions) -add_compile_options(-fPIC) add_compile_options(-I${CMAKE_CURRENT_SOURCE_DIR}/Common) add_compile_options(-I${CMAKE_CURRENT_BINARY_DIR}/Common) add_compile_options(-g) @@ -29,9 +28,19 @@ endif() add_compile_options(-Werror) if(CMAKE_SYSTEM_NAME STREQUAL Emscripten) + # Build a static library so no -fPIC set(CLR_CMAKE_PLATFORM_WASM 1) add_definitions(-D_WASM_) + # The emscripten build has additional warnings so -Werror breaks + add_compile_options(-Wno-unused-parameter) + add_compile_options(-Wno-unused-function) + add_compile_options(-Wno-alloca) + add_compile_options(-Wno-implicit-int-float-conversion) +else() + add_compile_options(-fPIC) + set(GEN_SHARED_LIB 1) endif(CMAKE_SYSTEM_NAME STREQUAL Emscripten) + if (CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64 OR CMAKE_SYSTEM_PROCESSOR STREQUAL amd64) add_definitions(-DBIT64=1) add_definitions(-D_AMD64_) @@ -95,7 +104,8 @@ else () message(FATAL_ERROR "Unknown build type. Set CMAKE_BUILD_TYPE to DEBUG or RELEASE.") endif () -if (APPLE) +if(CMAKE_SYSTEM_NAME STREQUAL Emscripten) +elseif (APPLE) add_definitions(-D__APPLE_USE_RFC_3542) # We cannot enable "stack-protector-strong" on OS X due to a bug in clang compiler (current version 7.0.2) diff --git a/src/Native/Unix/Common/pal_config.h.in b/src/Native/Unix/Common/pal_config.h.in index 6911969a0b88e..23cbd90f93dfa 100644 --- a/src/Native/Unix/Common/pal_config.h.in +++ b/src/Native/Unix/Common/pal_config.h.in @@ -67,6 +67,7 @@ #cmakedefine01 HAVE_SYS_SYSCTL_H #cmakedefine01 HAVE_NET_IFMEDIA_H #cmakedefine01 HAVE_LINUX_RTNETLINK_H +#cmakedefine01 HAVE_LINUX_CAN_H #cmakedefine01 HAVE_GETDOMAINNAME_SIZET #cmakedefine01 HAVE_INOTIFY #cmakedefine01 HAVE_CLOCK_MONOTONIC diff --git a/src/Native/Unix/System.Native/CMakeLists.txt b/src/Native/Unix/System.Native/CMakeLists.txt index 376ac24689533..73e8ccdf8dd12 100644 --- a/src/Native/Unix/System.Native/CMakeLists.txt +++ b/src/Native/Unix/System.Native/CMakeLists.txt @@ -31,12 +31,25 @@ if (CMAKE_SYSTEM_NAME STREQUAL Linux) endif () endif () -add_library(System.Native - SHARED - ${NATIVE_SOURCES} - ${VERSION_FILE_PATH} -) +if (GEN_SHARED_LIB) + add_library(System.Native + SHARED + ${NATIVE_SOURCES} + ${VERSION_FILE_PATH} + ) + if (CMAKE_SYSTEM_NAME STREQUAL Linux AND NOT CLR_CMAKE_PLATFORM_ANDROID) + target_link_libraries(System.Native rt) + endif () + if (CMAKE_SYSTEM_NAME STREQUAL FreeBSD) + target_link_libraries(System.Native pthread) + if (HAVE_INOTIFY) + find_library(INOTIFY_LIBRARY inotify HINTS /usr/local/lib) + target_link_libraries(System.Native ${INOTIFY_LIBRARY}) + endif () + endif () + install_library_and_symbols (System.Native) +endif () add_library(System.Native-Static STATIC @@ -48,17 +61,4 @@ add_library(System.Native-Static set_target_properties(System.Native-Static PROPERTIES PREFIX "") set_target_properties(System.Native-Static PROPERTIES OUTPUT_NAME System.Native CLEAN_DIRECT_OUTPUT 1) -if (CMAKE_SYSTEM_NAME STREQUAL Linux AND NOT CLR_CMAKE_PLATFORM_ANDROID) - target_link_libraries(System.Native rt) -endif () - -if (CMAKE_SYSTEM_NAME STREQUAL FreeBSD) - target_link_libraries(System.Native pthread) -if (HAVE_INOTIFY) - find_library(INOTIFY_LIBRARY inotify HINTS /usr/local/lib) - target_link_libraries(System.Native ${INOTIFY_LIBRARY}) -endif() -endif () - -install_library_and_symbols (System.Native) install (TARGETS System.Native-Static DESTINATION .) diff --git a/src/Native/Unix/System.Native/pal_networking.c b/src/Native/Unix/System.Native/pal_networking.c index 1ece3b648780c..8ee0569eefa93 100644 --- a/src/Native/Unix/System.Native/pal_networking.c +++ b/src/Native/Unix/System.Native/pal_networking.c @@ -55,7 +55,7 @@ #include #endif #endif -#ifdef AF_CAN +#if HAVE_LINUX_CAN_H #include #endif #if HAVE_KQUEUE @@ -2001,7 +2001,7 @@ static bool TryConvertProtocolTypePalToPlatform(int32_t palAddressFamily, int32_ *platformProtocolType = palProtocolType; return true; #endif -#ifdef AF_CAN +#if HAVE_LINUX_CAN_H case AddressFamily_AF_CAN: switch (palProtocolType) { @@ -2492,8 +2492,7 @@ static int32_t WaitForSocketEventsInner(int32_t port, SocketEvent* buffer, int32 } #else -#warning epoll/kqueue not detected; building with stub socket events support -static const size_t SocketEventBufferElementSize = sizeof(struct pollfd); +static const size_t SocketEventBufferElementSize = 0; static SocketEvents GetSocketEvents(int16_t filter, uint16_t flags) { diff --git a/src/Native/Unix/System.Native/pal_process.c b/src/Native/Unix/System.Native/pal_process.c index ffdb532c6b21e..97795517968ac 100644 --- a/src/Native/Unix/System.Native/pal_process.c +++ b/src/Native/Unix/System.Native/pal_process.c @@ -154,7 +154,7 @@ static int compare_groups(const void * a, const void * b) static int SetGroups(uint32_t* userGroups, int32_t userGroupsLength, uint32_t* processGroups) { -#ifdef __linux__ +#if defined(__linux__) || defined(_WASM_) size_t platformGroupsLength = Int32ToSizeT(userGroupsLength); #else // BSD int platformGroupsLength = userGroupsLength; diff --git a/src/Native/Unix/configure.cmake b/src/Native/Unix/configure.cmake index be2e227032222..913f1f98eca99 100644 --- a/src/Native/Unix/configure.cmake +++ b/src/Native/Unix/configure.cmake @@ -655,6 +655,10 @@ check_include_files( linux/rtnetlink.h HAVE_LINUX_RTNETLINK_H) +check_include_files( + linux/can.h + HAVE_LINUX_CAN_H) + check_symbol_exists( getpeereid unistd.h diff --git a/src/Native/Unix/gen-buildsys-clang.sh b/src/Native/Unix/gen-buildsys-clang.sh index 3bb7f7b899e4a..94690b60fe993 100755 --- a/src/Native/Unix/gen-buildsys-clang.sh +++ b/src/Native/Unix/gen-buildsys-clang.sh @@ -44,6 +44,7 @@ else buildtype="$5" fi +cmake_cmd=cmake cmake_extra_defines="-DCMAKE_BUILD_TYPE=$buildtype" if [[ -n "$CROSSCOMPILE" ]]; then if ! [[ -n "$ROOTFS_DIR" ]]; then @@ -60,6 +61,13 @@ fi if [ "$build_arch" == "armel" ]; then cmake_extra_defines="$cmake_extra_defines -DARM_SOFTFP=1" fi +if [ "$build_arch" == "wasm" ]; then + if [ "$EMSCRIPTEN_ROOT" == "" ]; then + EMSCRIPTEN_ROOT="$EMSDK_PATH/upstream/emscripten" + fi + cmake_cmd="emcmake cmake" + cmake_extra_defines="$cmake_extra_defines -DCMAKE_TOOLCHAIN_FILE=$EMSCRIPTEN_ROOT/cmake/Modules/Platform/Emscripten.cmake -DEMSCRIPTEN_GENERATE_BITCODE_STATIC_LIBRARIES=1" +fi __UnprocessedCMakeArgs="" if [ -z "$6" ]; then @@ -68,6 +76,7 @@ else __UnprocessedCMakeArgs="$6" fi -cmake $cmake_extra_defines \ +echo "Invoking \"$cmake_cmd $cmake_extra_defines $__UnprocessedCMakeArgs $1\"" +$cmake_cmd $cmake_extra_defines \ $__UnprocessedCMakeArgs \ $1 diff --git a/src/Native/Windows/gen-buildsys-win.bat b/src/Native/Windows/gen-buildsys-win.bat index 553ee889e8d38..683e25ca67730 100644 --- a/src/Native/Windows/gen-buildsys-win.bat +++ b/src/Native/Windows/gen-buildsys-win.bat @@ -36,6 +36,14 @@ popd :DoGen if "%3" == "wasm" ( + if "%EMSDK_PATH%" == "" ( + echo "Error: Should set EMSDK_PATH environment variable pointing to emsdk root." + exit /B 1 + ) + + if "%EMSCRIPTEN_ROOT%" == "" ( + set EMSCRIPTEN_ROOT="%EMSDK_PATH/upstream/emscripten%" + ) emcmake cmake "-DEMSCRIPTEN_GENERATE_BITCODE_STATIC_LIBRARIES=1" "-DCMAKE_TOOLCHAIN_FILE=%EMSCRIPTEN%/cmake/Modules/Platform/Emscripten.cmake" "-DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE%" -G "NMake Makefiles" %__sourceDir% ) else ( "%CMakePath%" %__SDKVersion% "-DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE%" "-DCMAKE_INSTALL_PREFIX=%__CMakeBinDir%" -G "Visual Studio %__VSString%" -B. -H%1 %__ExtraCmakeParams% diff --git a/src/Native/build-native.cmd b/src/Native/build-native.cmd index 3816e3737e91a..b90797f9f5279 100644 --- a/src/Native/build-native.cmd +++ b/src/Native/build-native.cmd @@ -160,10 +160,6 @@ IF ERRORLEVEL 1 ( goto :Failure ) -:: Copy results to native_aot since packaging expects a copy there too -mkdir "%__artifactsDir%\bin\native\%__outConfig%-aot" -copy "%__artifactsDir%\bin\native\%__outConfig%\*" "%__artifactsDir%\bin\native\%__outConfig%-aot" - exit /B 0 :Failure diff --git a/src/Native/build-native.proj b/src/Native/build-native.proj index 507ec1861aeef..69fbc27445dc7 100644 --- a/src/Native/build-native.proj +++ b/src/Native/build-native.proj @@ -20,7 +20,7 @@ then we should consider calling Environment.ProcessorCount --> <_ProcessorCountArg> --numproc $(MSBuildNodeCount) - <_StripSymbolsArg Condition="'$(BuildNativeStripSymbols)' == 'true'"> stripsymbols + <_StripSymbolsArg Condition="'$(BuildNativeStripSymbols)' == 'true' and '$(OSGroup)' != 'WebAssembly'"> stripsymbols <_PortableBuildArg Condition="'$(PortableBuild)' == 'true'"> -portable