Migrate to zlib-ng, part 2: consume it in runtime (second attempt) (#…

…104454)

* Reapply "Migrate to zlib-ng, part 2: consume it in runtime (#102403)" (#104414)
* Apply jkotas comment suggestion in configureplatform.cmake
* Delete unnecessary comment in zlib-ng.cmake
* Fix windows nativeaot failure happening when executing:

build.cmd -ci -arch x64 -os windows  -s clr.nativeaotlibs+clr.nativeaotruntime+libs+packs -c Release /p:BuildNativeAOTRuntimePack=true /p:SkipLibrariesNativeRuntimePackages=true

Directory.Build.props

@@ -324,6 +324,7 @@
     <NoTargetFrameworkFiltering>true</NoTargetFrameworkFiltering>
 
     <NativeBuildPartitionPropertiesToRemove>ClrFullNativeBuild;ClrRuntimeSubset;ClrJitSubset;ClrPalTestsSubset;ClrAllJitsSubset;ClrILToolsSubset;ClrNativeAotSubset;ClrSpmiSubset;ClrCrossComponentsSubset;ClrDebugSubset;HostArchitecture;PgoInstrument;NativeOptimizationDataSupported;CMakeArgs;CxxStandardLibrary;CxxStandardLibraryStatic;CxxAbiLibrary</NativeBuildPartitionPropertiesToRemove>
+    <UseSystemZlib Condition="'$(TargetsAppleMobile)' == 'true' or '$(TargetOS)' == 'android' or '$(TargetArchitecture)' == 'armv6'">true</UseSystemZlib>
   </PropertyGroup>
 
   <!-- RepositoryEngineeringDir isn't set when Installer tests import this file. -->

eng/native/configurecompiler.cmake

@@ -12,6 +12,11 @@ set(CMAKE_C_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_STANDARD 11)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
+# We need to set this to Release as there's no way to intercept configuration-specific linker flags
+# for try_compile-style tests (like check_c_source_compiles) and some of the default Debug flags
+# (ie. /INCREMENTAL) conflict with our own flags.
+set(CMAKE_TRY_COMPILE_CONFIGURATION Release)
+
 include(CheckCCompilerFlag)
 include(CheckCXXCompilerFlag)
 include(CheckLinkerFlag)
@@ -58,6 +63,7 @@ if (MSVC)
   define_property(TARGET PROPERTY CLR_CONTROL_FLOW_GUARD INHERITED BRIEF_DOCS "Controls the /guard:cf flag presence" FULL_DOCS "Set this property to ON or OFF to indicate if the /guard:cf compiler and linker flag should be present")
   define_property(TARGET PROPERTY CLR_EH_CONTINUATION INHERITED BRIEF_DOCS "Controls the /guard:ehcont flag presence" FULL_DOCS "Set this property to ON or OFF to indicate if the /guard:ehcont compiler flag should be present")
   define_property(TARGET PROPERTY CLR_EH_OPTION INHERITED BRIEF_DOCS "Defines the value of the /EH option" FULL_DOCS "Set this property to one of the valid /EHxx options (/EHa, /EHsc, /EHa-, ...)")
+  define_property(TARGET PROPERTY MSVC_WARNING_LEVEL INHERITED BRIEF_DOCS "Define the warning level for the /Wn option" FULL_DOCS "Set this property to one of the valid /Wn options (/W0, /W1, /W2, /W3, /W4)")
 
   set_property(GLOBAL PROPERTY CLR_CONTROL_FLOW_GUARD ON)
 
@@ -779,7 +785,8 @@ if (MSVC)
 
   # [[! Microsoft.Security.SystemsADM.10086 !]] - SDL required warnings
   # set default warning level to 4 but allow targets to override it.
-  add_compile_options($<$<COMPILE_LANGUAGE:C,CXX>:/W$<GENEX_EVAL:$<IF:$<BOOL:$<TARGET_PROPERTY:MSVC_WARNING_LEVEL>>,$<TARGET_PROPERTY:MSVC_WARNING_LEVEL>,4>>>)
+  set_property(GLOBAL PROPERTY MSVC_WARNING_LEVEL 4)
+  add_compile_options($<$<COMPILE_LANGUAGE:C,CXX>:/W$<TARGET_PROPERTY:MSVC_WARNING_LEVEL>>)
   add_compile_options($<$<COMPILE_LANGUAGE:C,CXX>:/WX>) # treat warnings as errors
   add_compile_options($<$<COMPILE_LANGUAGE:C,CXX>:/Oi>) # enable intrinsics
   add_compile_options($<$<COMPILE_LANGUAGE:C,CXX>:/Oy->) # disable suppressing of the creation of frame pointers on the call stack for quicker function calls

eng/native/configureplatform.cmake

@@ -500,3 +500,10 @@ if(LOWERCASE_CMAKE_BUILD_TYPE STREQUAL debug)
     string(REPLACE "-D_FORTIFY_SOURCE=2 " "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
     string(REPLACE "-D_FORTIFY_SOURCE=2 " "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
 endif()
+
+if (CLR_CMAKE_TARGET_ANDROID OR CLR_CMAKE_TARGET_MACCATALYST OR CLR_CMAKE_TARGET_IOS OR CLR_CMAKE_TARGET_TVOS OR CLR_CMAKE_HOST_ARCH_ARMV6)
+    # Some platforms are opted-out from using the in-tree zlib-ng by default:
+    # - Android and iOS-like platforms: concerns about extra binary size
+    # - Armv6: zlib-ng has build breaks
+    set(CLR_CMAKE_USE_SYSTEM_ZLIB 1)
+endif()

eng/native/functions.cmake

@@ -221,7 +221,7 @@ function(preprocess_file inputFilename outputFilename)
   get_compile_definitions(PREPROCESS_DEFINITIONS)
   get_include_directories(PREPROCESS_INCLUDE_DIRECTORIES)
   get_source_file_property(SOURCE_FILE_DEFINITIONS ${inputFilename} COMPILE_DEFINITIONS)
-  
+
   foreach(DEFINITION IN LISTS SOURCE_FILE_DEFINITIONS)
     list(APPEND PREPROCESS_DEFINITIONS -D${DEFINITION})
   endforeach()
@@ -508,7 +508,7 @@ function(install_static_library targetName destination component)
   if (WIN32)
     set_target_properties(${targetName} PROPERTIES
         COMPILE_PDB_NAME "${targetName}"
-        COMPILE_PDB_OUTPUT_DIRECTORY "${PROJECT_BINARY_DIR}"
+        COMPILE_PDB_OUTPUT_DIRECTORY "$<TARGET_FILE_DIR:${targetName}>"
     )
     install (FILES "$<TARGET_FILE_DIR:${targetName}>/${targetName}.pdb" DESTINATION ${destination} COMPONENT ${component})
   endif()

src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets

@@ -24,6 +24,7 @@ The .NET Foundation licenses this file to you under the MIT license.
     <LinkerFlavor Condition="'$(LinkerFlavor)' == '' and '$(_linuxLibcFlavor)' == 'bionic'">lld</LinkerFlavor>
     <LinkerFlavor Condition="'$(LinkerFlavor)' == '' and '$(_targetOS)' == 'linux'">bfd</LinkerFlavor>
     <IlcDefaultStackSize Condition="'$(IlcDefaultStackSize)' == '' and '$(_linuxLibcFlavor)' == 'musl'">1572864</IlcDefaultStackSize>
+    <UseSystemZlib Condition="!Exists('$(IlcSdkPath)libz.a')">true</UseSystemZlib>
   </PropertyGroup>
 
   <Target Name="SetupOSSpecificProps" DependsOnTargets="$(IlcDynamicBuildPropertyDependencies)">
@@ -129,6 +130,7 @@ The .NET Foundation licenses this file to you under the MIT license.
       <NativeLibrary Condition="'$(_targetArchitecture)' == 'x64'" Include="$(IlcSdkPath)$(VxSortSupportName)$(LibFileExt)" />
       <NativeLibrary Include="$(IlcSdkPath)$(StandaloneGCSupportName)$(LibFileExt)" />
       <NativeLibrary Condition="'$(LinkStandardCPlusPlusLibrary)' != 'true' and '$(StaticICULinking)' != 'true'" Include="$(IlcSdkPath)libstdc++compat.a" />
+      <NativeLibrary Condition="'$(UseSystemZlib)' != 'true'" Include="$(IlcSdkPath)libz.a" />
     </ItemGroup>
 
     <ItemGroup>
@@ -188,7 +190,7 @@ The .NET Foundation licenses this file to you under the MIT license.
       <NativeSystemLibrary Include="objc" Condition="'$(_IsApplePlatform)' == 'true'" />
       <NativeSystemLibrary Include="swiftCore" Condition="'$(_targetOS)' == 'osx'" />
       <NativeSystemLibrary Include="swiftFoundation" Condition="'$(_targetOS)' == 'osx'" />
-      <NativeSystemLibrary Include="z" />
+      <NativeSystemLibrary Include="z" Condition="'$(UseSystemZlib)' == 'true'" />
       <NativeSystemLibrary Include="rt" Condition="'$(_IsApplePlatform)' != 'true' and '$(_linuxLibcFlavor)' != 'bionic'" />
       <NativeSystemLibrary Include="log" Condition="'$(_linuxLibcFlavor)' == 'bionic'" />
       <NativeSystemLibrary Include="icucore" Condition="'$(_IsApplePlatform)' == 'true'" />

src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Windows.targets

@@ -46,6 +46,7 @@ The .NET Foundation licenses this file to you under the MIT license.
       <NativeLibrary Condition="'$(_targetArchitecture)' == 'x64'" Include="$(IlcSdkPath)$(VxSortSupportName)$(LibrarySuffix)" />
       <NativeLibrary Condition="'$(IlcMultiModule)' == 'true'" Include="$(SharedLibrary)" />
       <NativeLibrary Include="$(IlcSdkPath)$(StandaloneGCSupportName)$(LibrarySuffix)" />
+      <NativeLibrary Include="$(IlcSdkPath)zlibstatic$(LibFileExt)" />
     </ItemGroup>
 
     <ItemGroup>

src/installer/pkg/sfx/Microsoft.NETCore.App/Directory.Build.props

@@ -277,6 +277,9 @@
     <PlatformManifestFileEntry Include="libicudata.a" IsNative="true" />
     <PlatformManifestFileEntry Include="libicui18n.a" IsNative="true" />
     <PlatformManifestFileEntry Include="libicuuc.a" IsNative="true" />
+    <!-- zlib-specific files -->
+    <PlatformManifestFileEntry Include="libz.a" IsNative="true" />
+    <PlatformManifestFileEntry Include="zlibstatic.lib" IsNative="true" />
   </ItemGroup>
 
  <ItemGroup>

src/mono/CMakeLists.txt

@@ -249,7 +249,6 @@ elseif(CLR_CMAKE_HOST_OS STREQUAL "emscripten")
   set(DISABLE_SHARED_LIBS 1)
   # sys/random.h exists, but its not found
   set(HAVE_SYS_RANDOM_H 1)
-  set(INTERNAL_ZLIB 1)
 elseif(CLR_CMAKE_HOST_OS STREQUAL "wasi")
   set(HOST_WASI 1)
   add_definitions(-D_WASI_EMULATED_PROCESS_CLOCKS -D_WASI_EMULATED_SIGNAL -D_WASI_EMULATED_MMAN -DHOST_WASI)
@@ -261,7 +260,6 @@ elseif(CLR_CMAKE_HOST_OS STREQUAL "wasi")
   add_compile_options(-Wno-unused-but-set-variable)
   set(ENABLE_PERFTRACING 0)
   set(DISABLE_SHARED_LIBS 1)
-  set(INTERNAL_ZLIB 1)
   set(DISABLE_EXECUTABLES 1)
   set(STATIC_COMPONENTS 1)
 elseif(CLR_CMAKE_HOST_OS STREQUAL "windows")
@@ -270,7 +268,6 @@ elseif(CLR_CMAKE_HOST_OS STREQUAL "windows")
   set(HOST_NO_SYMLINKS 1)
   set(MONO_KEYWORD_THREAD "__declspec (thread)")
   set(MONO_ZERO_LEN_ARRAY 1)
-  set(INTERNAL_ZLIB 1)
   set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>") # statically link VC runtime library
   add_compile_options($<$<COMPILE_LANGUAGE:C,CXX>:/W4>)      # set warning level 4
   add_compile_options($<$<COMPILE_LANGUAGE:C,CXX>:/WX>)      # treat warnings as errors
@@ -511,16 +508,15 @@ if ((HOST_MACCAT AND HOST_ARM64) OR (TARGET_MACCAT AND TARGET_ARM64))
   set(TARGET_APPLE_MOBILE 1)
 endif()
 
-# Decide if we need zlib, and if so whether we want the system zlib or the in-tree copy.
+# Decide if we need zlib-ng.
 if(NOT DISABLE_EMBEDDED_PDB OR NOT DISABLE_LOG_PROFILER_GZ)
-  if(INTERNAL_ZLIB)
-    # defines ZLIB_SOURCES
-    include(${CLR_SRC_NATIVE_DIR}/external/zlib.cmake)
-  else()
-    # if we're not on a platform where we use the in-tree zlib, require system zlib
+  if (CLR_CMAKE_USE_SYSTEM_ZLIB)
+    # if we're not on a platform where we use the in-tree zlib-ng, require system zlib
     include(${CLR_SRC_NATIVE_DIR}/libs/System.IO.Compression.Native/extra_libs.cmake)
     set(Z_LIBS)
     append_extra_compression_libs(Z_LIBS)
+  else()
+    include(${CLR_SRC_NATIVE_DIR}/external/zlib-ng.cmake)
   endif()
 endif()
 
@@ -664,12 +660,17 @@ if(LLVM_PREFIX)
   endif()
   set(llvm_includedir "${LLVM_PREFIX}/include")
 
+  set(llvm_system_libs "")
   if(HOST_LINUX)
     # llvm-config --system-libs
-    set(llvm_system_libs ${MONO_cxx_lib} "-lz" "-lrt" "-ldl" "-lpthread" "-lm")
+    list(APPEND llvm_system_libs ${MONO_cxx_lib} "-lrt" "-ldl" "-lpthread" "-lm")
   elseif(HOST_OSX)
     # llvm-config --system-libs
-    set(llvm_system_libs "-lz" "-lm")
+    list(APPEND llvm_system_libs "-lm")
+  endif()
+
+  if (CLR_CMAKE_USE_SYSTEM_ZLIB AND (HOST_LINUX OR HOST_OSX))
+    list(APPEND llvm_system_libs "-lz")
   endif()
 
   # llvm-config --libs analysis core bitwriter mcjit orcjit

src/mono/browser/browser.proj

@@ -63,6 +63,7 @@
       <WasmPInvokeModule Include="libSystem.Native" />
       <WasmPInvokeModule Include="libSystem.IO.Compression.Native" />
       <WasmPInvokeModule Include="libSystem.Globalization.Native" />
+      <WasmPInvokeModule Include="libz" />
       <WasmPInvokeAssembly Include="@(LibrariesRuntimeFiles)" Condition="'%(Extension)' == '.dll' and '%(IsNative)' != 'true'" />
     </ItemGroup>
 

src/mono/browser/runtime/CMakeLists.txt

@@ -28,6 +28,7 @@ target_link_libraries(dotnet.native
     ${MONO_ARTIFACTS_DIR}/libmono-wasm-${CONFIGURATION_INTERPSIMDTABLES_LIB}.a
     ${MONO_ARTIFACTS_DIR}/libmono-profiler-aot.a
     ${MONO_ARTIFACTS_DIR}/libmono-profiler-browser.a
+    ${MONO_ARTIFACTS_DIR}/libz.a
     ${NATIVE_BIN_DIR}/wasm-bundled-timezones.a
     ${NATIVE_BIN_DIR}/libSystem.Native.a
     ${NATIVE_BIN_DIR}/libSystem.Globalization.Native.a

src/mono/cmake/config.h.in

@@ -168,9 +168,6 @@
 /* Define to 1 if you have the <unwind.h> header file. */
 #cmakedefine HAVE_UNWIND_H 1
 
-/* Use in-tree zlib */
-#cmakedefine INTERNAL_ZLIB 1
-
 /* Define to 1 if you have the <poll.h> header file. */
 #cmakedefine HAVE_POLL_H 1
 

src/mono/mono.proj

@@ -1189,6 +1189,9 @@ JS_ENGINES = [NODE_JS]
       <_MonoRuntimeArtifacts Condition="('$(TargetsBrowser)' == 'true'  or '$(TargetsWasi)' == 'true') and '$(BuildMonoAOTCrossCompilerOnly)' != 'true'" Include="$(MonoObjDir)out\lib\libmono-wasm-nosimd.a">
         <Destination>$(RuntimeBinDir)libmono-wasm-nosimd.a</Destination>
       </_MonoRuntimeArtifacts>
+      <_MonoRuntimeArtifacts Condition="('$(TargetsBrowser)' == 'true'  or '$(TargetsWasi)' == 'true') and '$(BuildMonoAOTCrossCompilerOnly)' != 'true'" Include="$(MonoObjDir)_deps\fetchzlibng-build\libz.a">
+        <Destination>$(RuntimeBinDir)libz.a</Destination>
+      </_MonoRuntimeArtifacts>
       <_MonoICorDebugArtifacts Condition="'$(MonoMsCorDbi)' == 'true'" Include="$(MonoObjDir)out\lib\$(LibPrefix)mscordbi$(LibSuffix)">
         <Destination>$(RuntimeBinDir)$(LibPrefix)mscordbi$(LibSuffix)</Destination>
       </_MonoICorDebugArtifacts>

src/mono/mono/eventpipe/test/CMakeLists.txt

@@ -39,7 +39,16 @@ if(ENABLE_PERFTRACING)
         set(CMAKE_SKIP_RPATH 1)
         add_executable(ep-test ${EVENTPIPE_TEST_SOURCES} ${EVENTPIPE_TEST_HEADERS})
         target_sources(ep-test PRIVATE "${mono-components-objects}")
-        target_link_libraries(ep-test PRIVATE eglib_api monosgen-static ${OS_LIBS} ${LLVM_LIBS} ${Z_LIBS} monoapi)
+
+        set(EPTEST_LINKABLE_LIBS "")
+        list(APPEND EPTEST_LINKABLE_LIBS eglib_api monosgen-static ${OS_LIBS} ${LLVM_LIBS} monoapi)
+        if (CLR_CMAKE_USE_SYSTEM_ZLIB)
+            list(APPEND EPTEST_LINKABLE_LIBS ${EPTEST_LINKABLE_LIBS} ${Z_LIBS})
+        else()
+            list(APPEND EPTEST_LINKABLE_LIBS ${EPTEST_LINKABLE_LIBS} zlib)
+        endif()
+        target_link_libraries(ep-test PRIVATE ${EPTEST_LINKABLE_LIBS})
+
         install_with_stripped_symbols(ep-test TARGETS bin)
     else(ENABLE_EVENTPIPE_TEST AND STATIC_COMPONENTS AND (NOT DISABLE_COMPONENTS) AND (NOT DISABLE_LIBS) AND (NOT DISABLE_EXECUTABLES))
         message(VERBOSE "Skip building native EventPipe library test runner.")

src/mono/mono/metadata/CMakeLists.txt

@@ -200,7 +200,14 @@ set(metadata_sources "${metadata_platform_sources};${metadata_common_sources};${
 if(HOST_WIN32 AND NOT DISABLE_SHARED_LIBS)
     add_library(metadata_objects_shared OBJECT ${metadata_sources})
     target_compile_definitions(metadata_objects_shared PRIVATE ${metadata_compile_definitions})
-    target_link_libraries(metadata_objects_shared PRIVATE monoapi eglib_api utils_objects_shared)
+
+    set(METADATAOBJECTSSHARED_LINKABLE_LIBS "")
+    list(APPEND METADATAOBJECTSSHARED_LINKABLE_LIBS monoapi eglib_api utils_objects_shared)
+    if (NOT CLR_CMAKE_USE_SYSTEM_ZLIB)
+        list(APPEND METADATAOBJECTSSHARED_LINKABLE_LIBS zlib)
+    endif()
+    target_link_libraries(metadata_objects_shared PRIVATE ${METADATAOBJECTSSHARED_LINKABLE_LIBS})
+
     # note: metadata_objects is an object library, so this doesn't force linking with sgen,
     # it just adds the relevant include directories - which we need even with Boehm
     target_link_libraries(metadata_objects_shared PRIVATE sgen_objects_shared)
@@ -212,7 +219,14 @@ endif()
 
 add_library(metadata_objects OBJECT ${metadata_sources})
 target_compile_definitions(metadata_objects PRIVATE ${metadata_compile_definitions})
-target_link_libraries(metadata_objects PRIVATE monoapi eglib_api utils_objects)
+
+set(METADATAOBJECTS_LINKABLE_LIBS "")
+list(APPEND METADATAOBJECTS_LINKABLE_LIBS monoapi eglib_api utils_objects)
+if (NOT CLR_CMAKE_USE_SYSTEM_ZLIB)
+    list(APPEND METADATAOBJECTS_LINKABLE_LIBS zlib)
+endif()
+target_link_libraries(metadata_objects PRIVATE ${METADATAOBJECTS_LINKABLE_LIBS})
+
 # note: metadata_objects is an object library, so this doesn't force linking with sgen,
 # it just adds the relevant include directories - which we need even with Boehm
 target_link_libraries(metadata_objects PRIVATE sgen_objects)

src/mono/mono/metadata/debug-mono-ppdb.c

@@ -30,12 +30,8 @@
 #include <mono/utils/mono-logger-internals.h>
 
 #ifndef DISABLE_EMBEDDED_PDB
-#ifdef INTERNAL_ZLIB
-#include <external/zlib/zlib.h>
-#else
 #include <zlib.h>
-#endif
-#endif
+#endif // DISABLE_EMBEDDED_PDB
 
 #include "debug-mono-ppdb.h"