[2018-08] [sdks] Create archive targets for pre-building on CI (#10932)

* [sdks] Remove redundant --enable-cxx

* [sdks] Shorten `-m32`/`-m64` parameters detection for runtimes

* [sdks] Build BCL profiles without configure flags

* [sdks] Add BclTemplate to build BCL profiles

This is to avoid building all the profiles in a single `sdks/out/bcl` directory which will be shared between android, ios and wasm.

* [sdks] Add download/pack capabilities for LLVM

* [sdks] Create archive targets for pre-building on CI

* [sdks] Use archive-{android,ios,wasm} on CI

* [sdks] Run archive-llvm-llvm{,win}{32,64} on CI

* [sdks] Ensure LLVM is cloned before trying to download

* [sdks] Simplify call to BclTemplate

* [mcs] Make directory dependency as order-only

We would otherwise always rebuild the target since the directory is updated whenever something is added to the directory. This is a common Makefile technique.

* [sdks] Build tests for BCL targets

mcs/build/tests.make

@@ -23,7 +23,7 @@ test_lib_dir = $(topdir)/class/lib/$(PROFILE)/tests
 
 test_nunit_lib = nunitlite.dll
 xunit_core := xunit.core xunit.execution.dotnet xunit.abstractions xunit.assert Xunit.NetCore.Extensions
-xunit_deps := netstandard System.Runtime
+xunit_deps := System.Runtime
 xunit_src  := $(patsubst %,$(topdir)/../external/xunit-binaries/%,BenchmarkAttribute.cs BenchmarkDiscover.cs) $(topdir)/../mcs/class/test-helpers/PlatformDetection.cs
 
 ifeq ($(USE_XTEST_REMOTE_EXECUTOR), YES)
@@ -249,7 +249,7 @@ ifdef HAVE_CS_TESTS
 $(test_lib_dir):
 	mkdir -p $@
 
-$(test_lib_output): $(test_assembly_dep) $(test_response) $(test_nunit_dep) $(test_lib_dir)
+$(test_lib_output): $(test_assembly_dep) $(test_response) $(test_nunit_dep) | $(test_lib_dir)
 	$(TEST_COMPILE) $(LIBRARY_FLAGS) -target:library -out:$@ $(test_flags) $(LOCAL_TEST_COMPILER_ONDOTNET_FLAGS) @$(test_response)
 
 test_response_preprocessed = $(test_response)_preprocessed
@@ -325,10 +325,10 @@ run-xunit-test-lib: xunit-test-local
 	@rm -f $(test_lib_dir)/xunit.execution.dotnet.dll
 
 # Some xunit tests want to be executed in a separate process (see RemoteExecutorTestBase)
-$(XTEST_REMOTE_EXECUTOR): $(topdir)/../external/corefx/src/Common/tests/System/Diagnostics/RemoteExecutorConsoleApp/RemoteExecutorConsoleApp.cs
+$(XTEST_REMOTE_EXECUTOR): $(topdir)/../external/corefx/src/Common/tests/System/Diagnostics/RemoteExecutorConsoleApp/RemoteExecutorConsoleApp.cs | $(test_lib_dir)
 	$(TEST_COMPILE) -r:$(topdir)/class/lib/$(PROFILE)/mscorlib.dll $< -out:$@
 
-$(xtest_lib_output): $(the_assembly) $(xtest_response) $(xunit_libs_dep) $(xunit_src) $(test_lib_dir) $(XTEST_REMOTE_EXECUTOR)
+$(xtest_lib_output): $(the_assembly) $(xtest_response) $(xunit_libs_dep) $(xunit_src) $(XTEST_REMOTE_EXECUTOR) | $(test_lib_dir)
 	$(TEST_COMPILE) $(LIBRARY_FLAGS) $(XTEST_LIB_FLAGS) -target:library -out:$@ $(xtest_flags) @$(xtest_response) $(xunit_src)
 
 xtest_response_preprocessed = $(xtest_response)_preprocessed

mcs/class/Microsoft.CSharp/monotouch_Microsoft.CSharp_xtest.dll.exclude.sources

@@ -0,0 +1 @@
+../../../external/corefx/src/Microsoft.CSharp/tests/AccessTests.netcoreapp.cs

mcs/class/Microsoft.CSharp/wasm_Microsoft.CSharp_xtest.dll.exclude.sources

@@ -0,0 +1 @@
+../../../external/corefx/src/Microsoft.CSharp/tests/AccessTests.netcoreapp.cs

mcs/class/Mono.Data.Sqlite/monotouch_Mono.Data.Sqlite_test.dll.exclude.sources

@@ -0,0 +1 @@
+Bug27864.cs

mcs/class/System.ComponentModel.Composition.4.5/monodroid_System.ComponentModel.Composition_xtest.dll.exclude.sources

@@ -0,0 +1,11 @@
+
+../../../external/corefx/src/System.ComponentModel.Composition/tests/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/ComponentModel/Composition/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/ComponentModel/Composition/AttributedModel/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/ComponentModel/Composition/Diagnostics/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/ComponentModel/Composition/Factories/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/ComponentModel/Composition/Primitives/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/ComponentModel/Composition/ReflectionModel/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/Integration/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/UnitTesting/*.cs

mcs/class/System.ComponentModel.Composition.4.5/monotouch_System.ComponentModel.Composition_xtest.dll.exclude.sources

@@ -0,0 +1,9 @@
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/ComponentModel/Composition/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/ComponentModel/Composition/AttributedModel/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/ComponentModel/Composition/Diagnostics/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/ComponentModel/Composition/Factories/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/ComponentModel/Composition/Primitives/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/ComponentModel/Composition/ReflectionModel/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/Integration/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/UnitTesting/*.cs

mcs/class/System.ComponentModel.Composition.4.5/wasm_System.ComponentModel.Composition_xtest.dll.exclude.sources

@@ -0,0 +1,9 @@
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/ComponentModel/Composition/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/ComponentModel/Composition/AttributedModel/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/ComponentModel/Composition/Diagnostics/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/ComponentModel/Composition/Factories/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/ComponentModel/Composition/Primitives/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/ComponentModel/Composition/ReflectionModel/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/Integration/*.cs
+../../../external/corefx/src/System.ComponentModel.Composition/tests/System/UnitTesting/*.cs

mcs/class/System.Core/monotouch_System.Core_xtest.dll.exclude.sources

@@ -0,0 +1 @@
+../../../external/corefx/src/System.Linq.Expressions/tests/Dynamic/InvokeMemberBindingTests.cs

mcs/class/System.Core/wasm_System.Core_xtest.dll.exclude.sources

@@ -0,0 +1 @@
+../../../external/corefx/src/System.Linq.Expressions/tests/Dynamic/InvokeMemberBindingTests.cs

mcs/class/System.Drawing/Makefile

@@ -17,7 +17,11 @@ ifndef MOBILE_PROFILE
 LIB_MCS_FLAGS += -d:FEATURE_TYPECONVERTER,SUPPORTS_WINDOWS_COLORS
 endif
 
-TEST_LIB_REFS = System.Drawing System.Runtime.Serialization.Formatters.Soap System.Xml
+TEST_LIB_REFS = System.Drawing System.Xml
+
+ifneq ($(PROFILE),wasm)
+TEST_LIB_REFS += System.Runtime.Serialization.Formatters.Soap
+endif
 
 TEST_MCS_FLAGS = $(LIB_MCS_FLAGS) -define:TEST -resource:Test/resources/indexed.png,indexed.png \
 	-nowarn:0618 -nowarn:219 -nowarn:169

mcs/class/System.Drawing/wasm_System.Drawing_test.dll.exclude.sources

@@ -0,0 +1,7 @@
+System.Drawing/GDIPlusTest.cs
+System.Drawing/TestBitmap.cs
+System.Drawing/TestGraphics.cs
+System.Drawing/TestIcon.cs
+System.Drawing/TestIconConverter.cs
+System.Drawing/TestImage.cs
+System.Drawing/TestImageConverter.cs

mcs/class/System.Runtime.Serialization/Makefile

@@ -13,7 +13,13 @@ LIB_MCS_FLAGS = \
 		$(RESOURCE_FILES:%=/resource:%)
 TXT_RESOURCE_STRINGS = ../referencesource/System.Runtime.Serialization/System.Runtime.Serialization.txt
 
-XTEST_LIB_REFS = System System.Core System.Xml System.Xml.Linq System.Drawing Facades/System.Threading.Tasks
+XTEST_LIB_REFS = System System.Core System.Xml System.Xml.Linq Facades/System.Threading.Tasks
+
+ifneq ($(PROFILE),monodroid)
+ifneq ($(PROFILE),monotouch)
+XTEST_LIB_REFS += System.Drawing
+endif
+endif
 
 ifneq (2.1, $(FRAMEWORK_VERSION))
 LIB_REFS += System.Data System.Configuration SMDiagnostics

mcs/class/System.Runtime.Serialization/monodroid_System.Runtime.Serialization_xtest.dll.exclude.sources

@@ -0,0 +1 @@
+../../../external/corefx/src/System.Runtime.Serialization.Xml/tests/SerializationTypes.RuntimeOnly.cs

mcs/class/System.Runtime.Serialization/monotouch_System.Runtime.Serialization_xtest.dll.exclude.sources

@@ -0,0 +1 @@
+../../../external/corefx/src/System.Runtime.Serialization.Xml/tests/SerializationTypes.RuntimeOnly.cs

mcs/class/System.Runtime.Serialization/wasm_System.Runtime.Serialization_xtest.dll.exclude.sources

@@ -0,0 +1 @@
+../../../external/corefx/src/System.Runtime.Serialization.Xml/tests/SerializationTypes.RuntimeOnly.cs

mcs/class/System.Security/wasm_System.Security_test.dll.exclude.sources

@@ -0,0 +1 @@
+System.Security.Permissions/DataProtectionPermissionAttributeTest.cs

mcs/class/System.Web.Services/monotouch_System.Web.Services_test.dll.exclude.sources

@@ -0,0 +1,8 @@
+System.Web.Services.Configuration/DiagnosticsElementTest.cs
+System.Web.Services.Configuration/ProtocolElementTest.cs
+System.Web.Services.Configuration/SoapEnvelopeProcessingElementTest.cs
+System.Web.Services.Configuration/SoapExtensionTypeElementTest.cs
+System.Web.Services.Configuration/TypeElementTest.cs
+System.Web.Services.Configuration/WsdlHelpGeneratorElementTest.cs
+System.Web.Services.Configuration/WsiProfilesElementTest.cs
+

mcs/class/System.Web.Services/wasm_System.Web.Services_test.dll.exclude.sources

@@ -0,0 +1,8 @@
+System.Web.Services.Configuration/DiagnosticsElementTest.cs
+System.Web.Services.Configuration/ProtocolElementTest.cs
+System.Web.Services.Configuration/SoapEnvelopeProcessingElementTest.cs
+System.Web.Services.Configuration/SoapExtensionTypeElementTest.cs
+System.Web.Services.Configuration/TypeElementTest.cs
+System.Web.Services.Configuration/WsdlHelpGeneratorElementTest.cs
+System.Web.Services.Configuration/WsiProfilesElementTest.cs
+

mcs/class/System.XML/Makefile

@@ -18,7 +18,13 @@ endif
 
 TEST_LIB_REFS = System.Data System.Core
 TEST_MCS_FLAGS = $(LIB_MCS_FLAGS) -nowarn:0618 -nowarn:219 -nowarn:169
-XTEST_LIB_REFS = System System.Core System.Xml System.Xml.Linq System.Drawing Facades/System.Threading.Tasks System.Runtime.Serialization Facades/System.Text.Encoding.CodePages
+XTEST_LIB_REFS = System System.Core System.Xml System.Xml.Linq Facades/System.Threading.Tasks System.Runtime.Serialization Facades/System.Text.Encoding.CodePages
+
+ifneq ($(PROFILE),monodroid)
+ifneq ($(PROFILE),monotouch)
+XTEST_LIB_REFS += System.Drawing
+endif
+endif
 
 LIBRARY_WARN_AS_ERROR = yes
 

mcs/class/System/monodroid_System_xtest.dll.exclude.sources

@@ -0,0 +1,15 @@
+
+../../../external/corefx/src/System.CodeDom/tests/CodeCollections/*.cs
+../../../external/corefx/src/System.CodeDom/tests/CodeExpressions/*.cs
+../../../external/corefx/src/System.CodeDom/tests/CodeObjects/*.cs
+../../../external/corefx/src/System.CodeDom/tests/CodeStatements/*.cs
+../../../external/corefx/src/System.CodeDom/tests/CodeTypeMembers/*.cs
+../../../external/corefx/src/System.CodeDom/tests/Compiler/*.cs
+../../../external/corefx/src/System.CodeDom/tests/Other/*.cs
+../../../external/corefx/src/System.CodeDom/tests/Microsoft/CSharp/*.cs
+../../../external/corefx/src/System.CodeDom/tests/Microsoft/VisualBasic/*.cs
+../../../external/corefx/src/System.CodeDom/tests/*.cs
+
+../../../external/corefx/src/System.Net.Security/tests/UnitTests/Fakes/*.cs
+
+../../../external/corefx/src/System.Text.RegularExpressions/tests/*.cs

mcs/class/System/monotouch_System_xtest.dll.exclude.sources

@@ -0,0 +1,10 @@
+../../../external/corefx/src/System.CodeDom/tests/CodeCollections/*.cs
+../../../external/corefx/src/System.CodeDom/tests/CodeExpressions/*.cs
+../../../external/corefx/src/System.CodeDom/tests/CodeObjects/*.cs
+../../../external/corefx/src/System.CodeDom/tests/CodeStatements/*.cs
+../../../external/corefx/src/System.CodeDom/tests/CodeTypeMembers/*.cs
+../../../external/corefx/src/System.CodeDom/tests/Compiler/*.cs
+../../../external/corefx/src/System.CodeDom/tests/Other/*.cs
+../../../external/corefx/src/System.CodeDom/tests/Microsoft/CSharp/*.cs
+../../../external/corefx/src/System.CodeDom/tests/Microsoft/VisualBasic/*.cs
+../../../external/corefx/src/System.CodeDom/tests/*.cs

mcs/class/System/wasm_System_xtest.dll.exclude.sources

@@ -0,0 +1,10 @@
+../../../external/corefx/src/System.CodeDom/tests/CodeCollections/*.cs
+../../../external/corefx/src/System.CodeDom/tests/CodeExpressions/*.cs
+../../../external/corefx/src/System.CodeDom/tests/CodeObjects/*.cs
+../../../external/corefx/src/System.CodeDom/tests/CodeStatements/*.cs
+../../../external/corefx/src/System.CodeDom/tests/CodeTypeMembers/*.cs
+../../../external/corefx/src/System.CodeDom/tests/Compiler/*.cs
+../../../external/corefx/src/System.CodeDom/tests/Other/*.cs
+../../../external/corefx/src/System.CodeDom/tests/Microsoft/CSharp/*.cs
+../../../external/corefx/src/System.CodeDom/tests/Microsoft/VisualBasic/*.cs
+../../../external/corefx/src/System.CodeDom/tests/*.cs

mcs/class/corlib/monotouch_corlib_xtest.dll.exclude.sources

@@ -0,0 +1,16 @@
+../../../external/corefx/src/System.Reflection.Emit/tests/MethodBuilder/MethodBuilderGetGenericArguments.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/MethodBuilder/MethodBuilderGetGenericMethodDefinition.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/MethodBuilder/MethodBuilderGetHashCode.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/MethodBuilder/MethodBuilderIsGenericMethod.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/MethodBuilder/MethodBuilderSetReturnType.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/MethodBuilder/MethodBuilderSetSignature.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/TypeBuilder/TypeBuilderDeclaringMethod.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/TypeBuilder/TypeBuilderGenericParameterAttributes.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/TypeBuilder/TypeBuilderGenericParameterPosition.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/TypeBuilder/TypeBuilderGetGenericTypeDefinition.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/TypeBuilder/TypeBuilderIsGenericTypeDefinition.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/TypeBuilder/TypeBuilderMakeGenericType.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/TypeBuilder/TypeBuilderName.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/TypeBuilder/TypeBuilderNamespace.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/TypeBuilder/TypeBuilderPackingSize.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/TypeBuilder/TypeBuilderSize.cs

mcs/class/corlib/wasm_corlib_xtest.dll.exclude.sources

@@ -0,0 +1,16 @@
+../../../external/corefx/src/System.Reflection.Emit/tests/MethodBuilder/MethodBuilderGetGenericArguments.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/MethodBuilder/MethodBuilderGetGenericMethodDefinition.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/MethodBuilder/MethodBuilderGetHashCode.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/MethodBuilder/MethodBuilderIsGenericMethod.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/MethodBuilder/MethodBuilderSetReturnType.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/MethodBuilder/MethodBuilderSetSignature.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/TypeBuilder/TypeBuilderDeclaringMethod.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/TypeBuilder/TypeBuilderGenericParameterAttributes.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/TypeBuilder/TypeBuilderGenericParameterPosition.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/TypeBuilder/TypeBuilderGetGenericTypeDefinition.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/TypeBuilder/TypeBuilderIsGenericTypeDefinition.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/TypeBuilder/TypeBuilderMakeGenericType.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/TypeBuilder/TypeBuilderName.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/TypeBuilder/TypeBuilderNamespace.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/TypeBuilder/TypeBuilderPackingSize.cs
+../../../external/corefx/src/System.Reflection.Emit/tests/TypeBuilder/TypeBuilderSize.cs

runtime/Makefile.am

@@ -1,3 +1,4 @@
+
 tmpinst = _tmpinst
 
 noinst_SCRIPTS = mono-wrapper monodis-wrapper
@@ -81,34 +82,49 @@ endif
 
 test_profiles = $(filter-out binary_reference_assemblies monodroid_tools,$(build_profiles))
 
-if BUILD_MCS
-
 MAKE_FLAGS=$(if $(V),,--no-print-directory -s)
 
 # The write check is to foil 'make distcheck'
-all-local: $(SUPPORT_FILES) $(TEST_SUPPORT_FILES)
+all-mcs: $(SUPPORT_FILES) $(TEST_SUPPORT_FILES)
 	if test -w $(mcs_topdir); then :; else chmod -R +w $(mcs_topdir); fi
 	cd $(mcs_topdir) && $(MAKE) $(MAKE_FLAGS) NO_DIR_CHECK=1 PROFILES='$(build_profiles)' CC='$(CC)' all-profiles
 
-# override automake
-install: install-exec install-data
+install-mcs: install-mcs-exec install-mcs-data
 
 # override automake
-install-exec: $(SUPPORT_FILES) $(TEST_SUPPORT_FILES)
+install-mcs-exec: $(SUPPORT_FILES) $(TEST_SUPPORT_FILES)
 	cd $(mcs_topdir) && $(MAKE) NO_DIR_CHECK=1 PROFILES='$(build_profiles)' RUNTIME_HAS_CONSISTENT_GACDIR=yes prefix=$(prefix) install-profiles
 
 # override automake
-install-data:
+install-mcs-data:
 	@:
 
 # override automake
-uninstall:
+uninstall-mcs:
 	cd $(mcs_topdir) && $(MAKE) NO_DIR_CHECK=1 PROFILES='$(build_profiles)' RUNTIME_HAS_CONSISTENT_GACDIR=yes prefix=$(prefix) uninstall-profiles
 
-clean-local:
+clean-mcs:
 	cd $(mcs_topdir) && $(MAKE) NO_DIR_CHECK=1 PROFILES='$(build_profiles)' clean-profiles
 	-rm -fr $(etctmp) $(tmpinst)
 
+if BUILD_MCS
+
+all-local: all-mcs
+
+# override automake
+install: install-mcs
+
+# override automake
+install-exec: install-mcs-exec
+
+# override automake
+install-data: install-mcs-data
+
+# override automake
+uninstall: uninstall-mcs
+
+clean-local: clean-mcs
+
 endif BUILD_MCS
 
 TEST_SUPPORT_FILES = $(tmpinst)/bin/mono $(tmpinst)/bin/ilasm $(tmpinst)/bin/csc $(tmpinst)/bin/mcs $(tmpinst)/bin/al