From f21e10deed466c71713e1b35dbc8a327880c2c7c Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Thu, 13 Jul 2023 04:52:42 -0400 Subject: [PATCH 1/2] [AndroidDependenciesTests] Test both manifest types (#8186) Updates the InstallAndroidDependenciesTest test to run against the self hosted Xamarin manifest, as well as Googles v2 manifest. For the default case, we shouldn't override any package versions. The dependency versions we calculate should be present in, and installable from, our manifest. For the Google manifest case, we should override the platform-tools version. Their manifest only ever contains one platform-tools version, and it is updated somewhat regularly causing the test to fail every time it changes. --- .../AndroidDependenciesTests.cs | 64 ++++++++++--------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AndroidDependenciesTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AndroidDependenciesTests.cs index 8da53fde3e3..b63d8c2102e 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AndroidDependenciesTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AndroidDependenciesTests.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text; using System.Xml; using System.Xml.Linq; using NUnit.Framework; @@ -16,11 +17,9 @@ public class AndroidDependenciesTests : BaseTest { [Test] [NonParallelizable] // Do not run environment modifying tests in parallel. - public void InstallAndroidDependenciesTest () + public void InstallAndroidDependenciesTest ([Values ("GoogleV2", "Xamarin")] string manifestType) { AssertCommercialBuild (); - // We need to grab the latest API level *before* changing env vars - var apiLevel = AndroidSdkResolver.GetMaxInstalledPlatform (); var oldSdkPath = Environment.GetEnvironmentVariable ("TEST_ANDROID_SDK_PATH"); var oldJdkPath = Environment.GetEnvironmentVariable ("TEST_ANDROID_JDK_PATH"); try { @@ -33,41 +32,48 @@ public void InstallAndroidDependenciesTest () Directory.Delete (path, recursive: true); Directory.CreateDirectory (path); } - var proj = new XamarinAndroidApplicationProject { - TargetSdkVersion = apiLevel.ToString (), + + var proj = new XamarinAndroidApplicationProject (); + var buildArgs = new List { + "AcceptAndroidSDKLicenses=true", + $"AndroidManifestType={manifestType}", }; - const string ExpectedPlatformToolsVersion = "34.0.4"; + // When using the default Xamarin manifest, this test should fail if we can't install any of the defaults in Xamarin.Android.Tools.Versions.props + // When using the Google manifest, override the platform tools version to the one in their manifest as it only ever contains one version + if (manifestType == "GoogleV2") { + buildArgs.Add ($"AndroidSdkPlatformToolsVersion={GetCurrentPlatformToolsVersion ()}"); + } + using (var b = CreateApkBuilder ()) { b.CleanupAfterSuccessfulBuild = false; string defaultTarget = b.Target; b.Target = "InstallAndroidDependencies"; b.BuildLogFile = "install-deps.log"; - Assert.IsTrue (b.Build (proj, parameters: new string [] { - "AcceptAndroidSDKLicenses=true", - "AndroidManifestType=GoogleV2", // Need GoogleV2 so we can install API-32 - $"AndroidSdkPlatformToolsVersion={ExpectedPlatformToolsVersion}", - }), "InstallAndroidDependencies should have succeeded."); - b.Target = defaultTarget; - b.BuildLogFile = "build.log"; - Assert.IsTrue (b.Build (proj, true), "build should have succeeded."); - bool usedNewDir = b.LastBuildOutput.ContainsText ($"Output Property: _AndroidSdkDirectory={sdkPath}"); - if (!usedNewDir) { - // Is this because the platform-tools version changed (again?!) - try { - var currentPlatformToolsVersion = GetCurrentPlatformToolsVersion (); - if (currentPlatformToolsVersion != ExpectedPlatformToolsVersion) { - Assert.Fail ($"_AndroidSdkDirectory not set to new SDK path `{sdkPath}`, *probably* because Google's repository has a newer platform-tools package! " + - $"repository2-3.xml contains platform-tools {currentPlatformToolsVersion}; expected {ExpectedPlatformToolsVersion}!"); + Assert.IsTrue (b.Build (proj, parameters: buildArgs.ToArray ()), "InstallAndroidDependencies should have succeeded."); + + // When dependencies can not be resolved/installed a warning will be present in build output: + // Dependency `platform-tools` should have been installed but could not be resolved. + var depFailedMessage = "should have been installed but could not be resolved"; + bool failedToInstall = b.LastBuildOutput.ContainsText (depFailedMessage); + if (failedToInstall) { + var sb = new StringBuilder (); + foreach (var line in b.LastBuildOutput) { + if (line.Contains (depFailedMessage)) { + sb.AppendLine (line); } } - catch (Exception e) { - TestContext.WriteLine ($"Could not extract platform-tools version from repository2-3.xml: {e}"); - } + Assert.Fail ($"A required dependency was not installed, warnings are listed below. Please check the task output in 'install-deps.log'.\n{sb.ToString ()}"); } - Assert.IsTrue (usedNewDir, $"_AndroidSdkDirectory was not set to new SDK path `{sdkPath}`."); + + b.Target = defaultTarget; + b.BuildLogFile = "build.log"; + Assert.IsTrue (b.Build (proj, true), "build should have succeeded."); + Assert.IsTrue ( b.LastBuildOutput.ContainsText ($"Output Property: _AndroidSdkDirectory={sdkPath}"), + $"_AndroidSdkDirectory was not set to new SDK path `{sdkPath}`. Please check the task output in 'install-deps.log'"); Assert.IsTrue (b.LastBuildOutput.ContainsText ($"Output Property: _JavaSdkDirectory={jdkPath}"), - $"_JavaSdkDirectory was not set to new JDK path `{jdkPath}`."); - Assert.IsTrue (b.LastBuildOutput.ContainsText ($"JavaPlatformJarPath={sdkPath}"), $"JavaPlatformJarPath did not contain new SDK path `{sdkPath}`."); + $"_JavaSdkDirectory was not set to new JDK path `{jdkPath}`. Please check the task output in 'install-deps.log'"); + Assert.IsTrue (b.LastBuildOutput.ContainsText ($"JavaPlatformJarPath={sdkPath}"), + $"JavaPlatformJarPath did not contain new SDK path `{sdkPath}`. Please check the task output in 'install-deps.log'"); } } finally { Environment.SetEnvironmentVariable ("TEST_ANDROID_SDK_PATH", oldSdkPath); @@ -89,7 +95,7 @@ static string GetCurrentPlatformToolsVersion () var revision = platformToolsPackage.Element ("revision"); - return $"{revision.Element ("major")}.{revision.Element ("minor")}.{revision.Element ("micro")}"; + return $"{revision.Element ("major")?.Value}.{revision.Element ("minor")?.Value}.{revision.Element ("micro")?.Value}"; } [Test] From 68368189d67c46ddbfed4e90e622f635c4aff11e Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 13 Jul 2023 19:59:28 +0000 Subject: [PATCH 2/2] [Xamarin.Android.Build.Tasks] per-RID assemblies & typemaps (#8164) Context: 929e7012410233e6814af369db582f238ba185ad Context: ce2bc689a19cb45f7d7bfdd371c16c54b018a020 Context: https://github.com/xamarin/xamarin-android/issues/7473 Context: https://github.com/xamarin/xamarin-android/issues/8155 The managed linker can produce assemblies optimized for the target `$(RuntimeIdentifier)` (RID), which means that they will differ between different RIDs. Our "favorite" example of this is `IntPtr.Size`, which is inlined by the linker into `4` or `8` when targeting 32-bit or 64-bit platforms. (See also #7473 and 929e7012.) Another platform difference may come in the shape of CPU intrinsics which will change the JIT-generated native code in ways that will crash the application if the assembler instructions generated for the intrinsics aren't supported by the underlying processor. In addition, the per-RID assemblies will have different [MVID][0]s and **may** have different type and method metadata token IDs, which is important because typemaps *use* type and metadata token IDs; see also ce2bc689. All of this taken together invalidates our previous assumption that all the managed assemblies are identical. "Simply" using `IntPtr.Size` in an assembly that contains `Java.Lang.Object` subclasses will break things. This in turn could cause "mysterious" behavior or crashes in Release applications; see also Issue #8155. Prevent the potential problems by processing each per-RID assembly separately and output correct per-RID LLVM IR assembly using the appropriate per-RID information. Additionally, during testing I found that for our use of Cecil within `` doesn't consistently remove the fields, delegates, and methods we remove in `MarshalMethodsAssemblyRewriter` when marshal methods are enabled, or it generates subtly broken assemblies which cause **some** applications to segfault at run time like so: I monodroid-gc: 1 outstanding GREFs. Performing a full GC! F libc : Fatal signal 11 (SIGSEGV), code 1 (SEGV_MAPERR), fault addr 0x8 in tid 12379 (t6.helloandroid), pid 12379 (t6.helloandroid) F DEBUG : *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** *** F DEBUG : Build fingerprint: 'google/raven_beta/raven:14/UPB3.230519.014/10284690:user/release-keys' F DEBUG : Revision: 'MP1.0' F DEBUG : ABI: 'arm64' F DEBUG : Timestamp: 2023-07-04 22:09:58.762982002+0200 F DEBUG : Process uptime: 1s F DEBUG : Cmdline: com.microsoft.net6.helloandroid F DEBUG : pid: 12379, tid: 12379, name: t6.helloandroid >>> com.microsoft.net6.helloandroid <<< F DEBUG : uid: 10288 F DEBUG : tagged_addr_ctrl: 0000000000000001 (PR_TAGGED_ADDR_ENABLE) F DEBUG : signal 11 (SIGSEGV), code 1 (SEGV_MAPERR), fault addr 0x0000000000000008 F DEBUG : Cause: null pointer dereference F DEBUG : x0 0000000000000000 x1 0000007ba1401af0 x2 00000000000000fa x3 0000000000000001 F DEBUG : x4 0000007ba1401b38 x5 0000007b9f2a8360 x6 0000000000000000 x7 0000000000000000 F DEBUG : x8 ffffffffffc00000 x9 0000007b9f800000 x10 0000000000000000 x11 0000007ba1400000 F DEBUG : x12 0000000000000000 x13 0000007ba374ad58 x14 0000000000000000 x15 00000013ead77d66 F DEBUG : x16 0000007ba372f210 x17 0000007ebdaa4a80 x18 0000007edf612000 x19 000000000000001f F DEBUG : x20 0000000000000000 x21 0000007b9f2a8320 x22 0000007b9fb02000 x23 0000000000000018 F DEBUG : x24 0000007ba374ad08 x25 0000000000000004 x26 0000007b9f2a4618 x27 0000000000000000 F DEBUG : x28 ffffffffffffffff x29 0000007fc592a780 F DEBUG : lr 0000007ba3701f44 sp 0000007fc592a730 pc 0000007ba3701e0c pst 0000000080001000 F DEBUG : 8 total frames F DEBUG : backtrace: F DEBUG : #00 pc 00000000002d4e0c /data/app/~~Av24J15xbf20XdrY3X2_wA==/com.microsoft.net6.helloandroid-4DusuNWIAkz1Ssi7fWVF-g==/lib/arm64/libmonosgen-2.0.so (BuildId: 761134f2369377582cc3a8e25ecccb43a2e0a877) F DEBUG : #01 pc 00000000002c29e8 /data/app/~~Av24J15xbf20XdrY3X2_wA==/com.microsoft.net6.helloandroid-4DusuNWIAkz1Ssi7fWVF-g==/lib/arm64/libmonosgen-2.0.so (BuildId: 761134f2369377582cc3a8e25ecccb43a2e0a877) F DEBUG : #02 pc 00000000002c34bc /data/app/~~Av24J15xbf20XdrY3X2_wA==/com.microsoft.net6.helloandroid-4DusuNWIAkz1Ssi7fWVF-g==/lib/arm64/libmonosgen-2.0.so (BuildId: 761134f2369377582cc3a8e25ecccb43a2e0a877) F DEBUG : #03 pc 00000000002c2254 /data/app/~~Av24J15xbf20XdrY3X2_wA==/com.microsoft.net6.helloandroid-4DusuNWIAkz1Ssi7fWVF-g==/lib/arm64/libmonosgen-2.0.so (BuildId: 761134f2369377582cc3a8e25ecccb43a2e0a877) F DEBUG : #04 pc 00000000002be0bc /data/app/~~Av24J15xbf20XdrY3X2_wA==/com.microsoft.net6.helloandroid-4DusuNWIAkz1Ssi7fWVF-g==/lib/arm64/libmonosgen-2.0.so (BuildId: 761134f2369377582cc3a8e25ecccb43a2e0a877) F DEBUG : #05 pc 00000000002bf050 /data/app/~~Av24J15xbf20XdrY3X2_wA==/com.microsoft.net6.helloandroid-4DusuNWIAkz1Ssi7fWVF-g==/lib/arm64/libmonosgen-2.0.so (BuildId: 761134f2369377582cc3a8e25ecccb43a2e0a877) F DEBUG : #06 pc 00000000002a53a4 /data/app/~~Av24J15xbf20XdrY3X2_wA==/com.microsoft.net6.helloandroid-4DusuNWIAkz1Ssi7fWVF-g==/lib/arm64/libmonosgen-2.0.so (mono_gc_collect+44) (BuildId: 761134f2369377582cc3a8e25ecccb43a2e0a877) F DEBUG : #07 pc 000000000000513c This is because we generate Java Callable Wrappers over a set of original (linked or not) assemblies, then we scan them for classes derived from `Java.Lang.Object` and use that set as input to the marshal methods rewriter, which makes the changes (generates wrapper methods, decorates wrapped methods with `[UnmanagedCallersOnly]`, removes the old delegate methods as well as delegate backing fields) to all the `Java.Lang.Object` subclasses, then writes the modified assembly to a `new/` location (efa14e26), followed by copying the newly written assemblies back to the original location. At this point, we have the results returned by the subclass scanner in memory and **new** versions of those types on disk, but they are out of sync, since the types in memory refer to the **old** assemblies, but AOT is ran on the **new** assemblies which have a different layout, changed MVIDs and, potentially, different type and method token IDs (because we added some methods, removed others etc) and thus it causes the crashes at the run time. The now invalid set of "old" types is passed to the typemap generator. This only worked by accident, because we (incorrectly) used only the first linked assembly which happened to be the same one passed to the JLO scanner and AOT - so everything was fine at the execution time. Address this by *disabling* LLVM Marshal Methods (8bc7a3e8) for .NET 8, setting `$(AndroidEnableMarshalMethods)`=False by default. We'll attempt to fix these issues for .NET 9. [0]: https://learn.microsoft.com/dotnet/api/system.reflection.module.moduleversionid?view=net-7.0 --- .../Tasks/GenerateJavaStubs.cs | 276 +++++++++++---- .../Xamarin.Android.Build.Tests/BuildTest.cs | 4 +- .../Xamarin.Android.Build.Tests/BuildTest2.cs | 2 +- .../Tasks/LinkerTests.cs | 54 ++- .../BuildReleaseArm64SimpleDotNet.apkdesc | 34 +- .../BuildReleaseArm64XFormsDotNet.apkdesc | 108 +++--- .../Utilities/ManifestDocument.cs | 18 +- .../MarshalMethodsAssemblyRewriter.cs | 101 +++--- .../Utilities/MarshalMethodsClassifier.cs | 26 +- .../Utilities/MonoAndroidHelper.cs | 21 ++ .../Utilities/TypeMapGenerator.cs | 251 +++++++++----- ...peMappingReleaseNativeAssemblyGenerator.cs | 129 ++++--- .../Utilities/XAAssemblyResolver.cs | 326 ++++++++++++++++++ .../Utilities/XAJavaTypeScanner.cs | 120 +++++++ .../Xamarin.Android.Common.targets | 2 +- .../Tests/InstallAndRunTests.cs | 5 + 16 files changed, 1123 insertions(+), 354 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 04286ff5d77..80f1f25b279 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.IO.MemoryMappedFiles; using System.Linq; using System.Reflection; using System.Text; @@ -103,7 +104,7 @@ public override bool RunTask () bool useMarshalMethods = !Debug && EnableMarshalMethods; // We're going to do 3 steps here instead of separate tasks so // we can share the list of JLO TypeDefinitions between them - using (DirectoryAssemblyResolver res = MakeResolver (useMarshalMethods)) { + using (XAAssemblyResolver res = MakeResolver (useMarshalMethods)) { Run (res, useMarshalMethods); } } catch (XamarinAndroidException e) { @@ -122,7 +123,7 @@ public override bool RunTask () return !Log.HasLoggedErrors; } - DirectoryAssemblyResolver MakeResolver (bool useMarshalMethods) + XAAssemblyResolver MakeResolver (bool useMarshalMethods) { var readerParams = new ReaderParameters(); if (useMarshalMethods) { @@ -130,31 +131,32 @@ DirectoryAssemblyResolver MakeResolver (bool useMarshalMethods) readerParams.InMemory = true; } - var res = new DirectoryAssemblyResolver (this.CreateTaskLogger (), loadDebugSymbols: true, loadReaderParameters: readerParams); + var res = new XAAssemblyResolver (Log, loadDebugSymbols: true, loadReaderParameters: readerParams); foreach (var dir in FrameworkDirectories) { if (Directory.Exists (dir.ItemSpec)) { - res.SearchDirectories.Add (dir.ItemSpec); + res.FrameworkSearchDirectories.Add (dir.ItemSpec); } } return res; } - void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) + void Run (XAAssemblyResolver res, bool useMarshalMethods) { PackageNamingPolicy pnp; JavaNativeTypeManager.PackageNamingPolicy = Enum.TryParse (PackageNamingPolicy, out pnp) ? pnp : PackageNamingPolicyEnum.LowercaseCrc64; - Dictionary> marshalMethodsAssemblyPaths = null; + Dictionary>? abiSpecificAssembliesByPath = null; if (useMarshalMethods) { - marshalMethodsAssemblyPaths = new Dictionary> (StringComparer.Ordinal); + abiSpecificAssembliesByPath = new Dictionary> (StringComparer.Ordinal); } // Put every assembly we'll need in the resolver bool hasExportReference = false; bool haveMonoAndroid = false; - var allTypemapAssemblies = new HashSet (StringComparer.OrdinalIgnoreCase); + var allTypemapAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); var userAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); + foreach (var assembly in ResolvedAssemblies) { bool value; if (bool.TryParse (assembly.GetMetadata (AndroidSkipJavaStubGeneration), out value) && value) { @@ -180,13 +182,13 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) } if (addAssembly) { - allTypemapAssemblies.Add (assembly.ItemSpec); + MaybeAddAbiSpecifcAssembly (assembly, fileName); + if (!allTypemapAssemblies.ContainsKey (assembly.ItemSpec)) { + allTypemapAssemblies.Add (assembly.ItemSpec, assembly); + } } - res.Load (assembly.ItemSpec); - if (useMarshalMethods) { - StoreMarshalAssemblyPath (Path.GetFileNameWithoutExtension (assembly.ItemSpec), assembly); - } + res.Load (MonoAndroidHelper.GetTargetArch (assembly), assembly.ItemSpec); } // However we only want to look for JLO types in user code for Java stub code generation @@ -195,31 +197,33 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) Log.LogDebugMessage ($"Skipping Java Stub Generation for {asm.ItemSpec}"); continue; } - if (!allTypemapAssemblies.Contains (asm.ItemSpec)) - allTypemapAssemblies.Add (asm.ItemSpec); + res.Load (MonoAndroidHelper.GetTargetArch (asm), asm.ItemSpec); + MaybeAddAbiSpecifcAssembly (asm, Path.GetFileName (asm.ItemSpec)); + if (!allTypemapAssemblies.ContainsKey (asm.ItemSpec)) { + allTypemapAssemblies.Add (asm.ItemSpec, asm); + } + string name = Path.GetFileNameWithoutExtension (asm.ItemSpec); if (!userAssemblies.ContainsKey (name)) userAssemblies.Add (name, asm.ItemSpec); - StoreMarshalAssemblyPath (name, asm); } // Step 1 - Find all the JLO types var cache = new TypeDefinitionCache (); - var scanner = new JavaTypeScanner (this.CreateTaskLogger (), cache) { + var scanner = new XAJavaTypeScanner (Log, cache) { ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, }; + List allJavaTypes = scanner.GetJavaTypes (allTypemapAssemblies.Values, res); + var javaTypes = new List (); - List allJavaTypes = scanner.GetJavaTypes (allTypemapAssemblies, res); - - var javaTypes = new List (); - foreach (TypeDefinition td in allJavaTypes) { + foreach (JavaType jt in allJavaTypes) { // Whem marshal methods are in use we do not want to skip non-user assemblies (such as Mono.Android) - we need to generate JCWs for them during // application build, unlike in Debug configuration or when marshal methods are disabled, in which case we use JCWs generated during Xamarin.Android // build and stored in a jar file. - if ((!useMarshalMethods && !userAssemblies.ContainsKey (td.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (td, cache)) { + if ((!useMarshalMethods && !userAssemblies.ContainsKey (jt.Type.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (jt.Type, cache)) { continue; } - javaTypes.Add (td); + javaTypes.Add (jt); } MarshalMethodsClassifier classifier = null; @@ -237,27 +241,11 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) // in order to properly generate wrapper methods in the marshal methods assembly rewriter. // We don't care about those generated by us, since they won't contain the `XA_BROKEN_EXCEPTION_TRANSITIONS` variable we look for. var environmentParser = new EnvironmentFilesParser (); - var targetPaths = new List (); - - if (!LinkingEnabled) { - targetPaths.Add (Path.GetDirectoryName (ResolvedAssemblies[0].ItemSpec)); - } else { - if (String.IsNullOrEmpty (IntermediateOutputDirectory)) { - throw new InvalidOperationException ($"Internal error: marshal methods require the `IntermediateOutputDirectory` property of the `GenerateJavaStubs` task to have a value"); - } - // If the property is set then, even if we have just one RID, the linked assemblies path will include the RID - if (!HaveMultipleRIDs && SupportedAbis.Length == 1) { - targetPaths.Add (Path.Combine (IntermediateOutputDirectory, "linked")); - } else { - foreach (string abi in SupportedAbis) { - targetPaths.Add (Path.Combine (IntermediateOutputDirectory, AbiToRid (abi), "linked")); - } - } - } + Dictionary assemblyPaths = AddMethodsFromAbiSpecificAssemblies (classifier, res, abiSpecificAssembliesByPath); - var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, marshalMethodsAssemblyPaths, Log); - rewriter.Rewrite (res, targetPaths, environmentParser.AreBrokenExceptionTransitionsEnabled (Environments)); + var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, assemblyPaths, Log); + rewriter.Rewrite (res, environmentParser.AreBrokenExceptionTransitionsEnabled (Environments)); } // Step 3 - Generate type maps @@ -272,7 +260,8 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) var javaConflicts = new Dictionary> (0, StringComparer.Ordinal); using (var acw_map = MemoryStreamPool.Shared.CreateStreamWriter ()) { - foreach (TypeDefinition type in javaTypes) { + foreach (JavaType jt in javaTypes) { + TypeDefinition type = jt.Type; string managedKey = type.FullName.Replace ('/', '.'); string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'); @@ -381,7 +370,8 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) // Create additional application java sources. StringWriter regCallsWriter = new StringWriter (); regCallsWriter.WriteLine ("\t\t// Application and Instrumentation ACWs must be registered first."); - foreach (var type in javaTypes) { + foreach (JavaType jt in javaTypes) { + TypeDefinition type = jt.Type; if (JavaNativeTypeManager.IsApplication (type, cache) || JavaNativeTypeManager.IsInstrumentation (type, cache)) { if (classifier != null && !classifier.FoundDynamicallyRegisteredMethods (type)) { continue; @@ -414,43 +404,55 @@ void Run (DirectoryAssemblyResolver res, bool useMarshalMethods) } } - void StoreMarshalAssemblyPath (string name, ITaskItem asm) + void MaybeAddAbiSpecifcAssembly (ITaskItem assembly, string fileName) { - if (!useMarshalMethods) { + if (abiSpecificAssembliesByPath == null) { return; } - // TODO: we need to keep paths to ALL the assemblies, we need to rewrite them for all RIDs eventually. Right now we rewrite them just for one RID - if (!marshalMethodsAssemblyPaths.TryGetValue (name, out HashSet assemblyPaths)) { - assemblyPaths = new HashSet (); - marshalMethodsAssemblyPaths.Add (name, assemblyPaths); - } + string? abi = assembly.GetMetadata ("Abi"); + if (!String.IsNullOrEmpty (abi)) { + if (!abiSpecificAssembliesByPath.TryGetValue (fileName, out List? items)) { + items = new List (); + abiSpecificAssembliesByPath.Add (fileName, items); + } - assemblyPaths.Add (asm.ItemSpec); + items.Add (assembly); + } } + } - string AbiToRid (string abi) - { - switch (abi) { - case "arm64-v8a": - return "android-arm64"; + AssemblyDefinition LoadAssembly (string path, XAAssemblyResolver? resolver = null) + { + string pdbPath = Path.ChangeExtension (path, ".pdb"); + var readerParameters = new ReaderParameters { + AssemblyResolver = resolver, + InMemory = false, + ReadingMode = ReadingMode.Immediate, + ReadSymbols = File.Exists (pdbPath), + ReadWrite = false, + }; - case "armeabi-v7a": - return "android-arm"; + MemoryMappedViewStream? viewStream = null; + try { + // Create stream because CreateFromFile(string, ...) uses FileShare.None which is too strict + using var fileStream = new FileStream (path, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, false); + using var mappedFile = MemoryMappedFile.CreateFromFile ( + fileStream, null, fileStream.Length, MemoryMappedFileAccess.Read, HandleInheritability.None, true); + viewStream = mappedFile.CreateViewStream (0, 0, MemoryMappedFileAccess.Read); - case "x86": - return "android-x86"; + AssemblyDefinition result = ModuleDefinition.ReadModule (viewStream, readerParameters).Assembly; - case "x86_64": - return "android-x64"; + // We transferred the ownership of the viewStream to the collection. + viewStream = null; - default: - throw new InvalidOperationException ($"Internal error: unsupported ABI '{abi}'"); - } + return result; + } finally { + viewStream?.Dispose (); } } - bool CreateJavaSources (IEnumerable javaTypes, TypeDefinitionCache cache, MarshalMethodsClassifier classifier, bool useMarshalMethods) + bool CreateJavaSources (IEnumerable newJavaTypes, TypeDefinitionCache cache, MarshalMethodsClassifier classifier, bool useMarshalMethods) { if (useMarshalMethods && classifier == null) { throw new ArgumentNullException (nameof (classifier)); @@ -462,7 +464,8 @@ bool CreateJavaSources (IEnumerable javaTypes, TypeDefinitionCac bool generateOnCreateOverrides = int.Parse (AndroidSdkPlatform) <= 10; bool ok = true; - foreach (var t in javaTypes) { + foreach (JavaType jt in newJavaTypes) { + TypeDefinition t = jt.Type; // JCW generator doesn't care about ABI-specific types or token ids if (t.IsInterface) { // Interfaces are in typemap but they shouldn't have JCW generated for them continue; @@ -565,13 +568,146 @@ void SaveResource (string resource, string filename, string destDir, Func types, TypeDefinitionCache cache) + void WriteTypeMappings (List types, TypeDefinitionCache cache) { var tmg = new TypeMapGenerator ((string message) => Log.LogDebugMessage (message), SupportedAbis); - if (!tmg.Generate (Debug, SkipJniAddNativeMethodRegistrationAttributeScan, types, cache, TypemapOutputDirectory, GenerateNativeAssembly, out ApplicationConfigTaskState appConfState)) + if (!tmg.Generate (Debug, SkipJniAddNativeMethodRegistrationAttributeScan, types, cache, TypemapOutputDirectory, GenerateNativeAssembly, out ApplicationConfigTaskState appConfState)) { throw new XamarinAndroidException (4308, Properties.Resources.XA4308); + } GeneratedBinaryTypeMaps = tmg.GeneratedBinaryTypeMaps.ToArray (); BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (ApplicationConfigTaskState.RegisterTaskObjectKey), appConfState, RegisteredTaskObjectLifetime.Build); } + + /// + /// + /// Classifier will see only unique assemblies, since that's what's processed by the JI type scanner - even though some assemblies may have + /// abi-specific features (e.g. inlined `IntPtr.Size` or processor-specific intrinsics), the **types** and **methods** will all be the same and, thus, + /// there's no point in scanning all of the additional copies of the same assembly. + /// + /// + /// This, however, doesn't work for the rewriter which needs to rewrite all of the copies so that they all have the same generated wrappers. In + /// order to do that, we need to go over the list of assemblies found by the classifier, see if they are abi-specific ones and then add all the + /// marshal methods from the abi-specific assembly copies, so that the rewriter can easily rewrite them all. + /// + /// + /// This method returns a dictionary matching `AssemblyDefinition` instances to the path on disk to the assembly file they were loaded from. It is necessary + /// because uses a stream to load the data, in order to avoid later sharing violation issues when writing the assemblies. Path + /// information is required by to be available for each + /// + /// + Dictionary AddMethodsFromAbiSpecificAssemblies (MarshalMethodsClassifier classifier, XAAssemblyResolver resolver, Dictionary> abiSpecificAssemblies) + { + IDictionary> marshalMethods = classifier.MarshalMethods; + ICollection assemblies = classifier.Assemblies; + var newAssemblies = new List (); + var assemblyPaths = new Dictionary (); + + foreach (AssemblyDefinition asmdef in assemblies) { + string fileName = Path.GetFileName (asmdef.MainModule.FileName); + if (!abiSpecificAssemblies.TryGetValue (fileName, out List? abiAssemblyItems)) { + continue; + } + + List assemblyMarshalMethods = FindMarshalMethodsForAssembly (marshalMethods, asmdef);; + Log.LogDebugMessage ($"Assembly {fileName} is ABI-specific"); + foreach (ITaskItem abiAssemblyItem in abiAssemblyItems) { + if (String.Compare (abiAssemblyItem.ItemSpec, asmdef.MainModule.FileName, StringComparison.Ordinal) == 0) { + continue; + } + + Log.LogDebugMessage ($"Looking for matching mashal methods in {abiAssemblyItem.ItemSpec}"); + FindMatchingMethodsInAssembly (abiAssemblyItem, classifier, assemblyMarshalMethods, resolver, newAssemblies, assemblyPaths); + } + } + + if (newAssemblies.Count > 0) { + foreach (AssemblyDefinition asmdef in newAssemblies) { + assemblies.Add (asmdef); + } + } + + return assemblyPaths; + } + + List FindMarshalMethodsForAssembly (IDictionary> marshalMethods, AssemblyDefinition asm) + { + var seenNativeCallbacks = new HashSet (); + var assemblyMarshalMethods = new List (); + + foreach (var kvp in marshalMethods) { + foreach (MarshalMethodEntry method in kvp.Value) { + if (method.NativeCallback.Module.Assembly != asm) { + continue; + } + + // More than one overriden method can use the same native callback method, we're interested only in unique native + // callbacks, since that's what gets rewritten. + if (seenNativeCallbacks.Contains (method.NativeCallback)) { + continue; + } + + seenNativeCallbacks.Add (method.NativeCallback); + assemblyMarshalMethods.Add (method); + } + } + + return assemblyMarshalMethods; + } + + void FindMatchingMethodsInAssembly (ITaskItem assemblyItem, MarshalMethodsClassifier classifier, List assemblyMarshalMethods, XAAssemblyResolver resolver, List newAssemblies, Dictionary assemblyPaths) + { + AssemblyDefinition asm = LoadAssembly (assemblyItem.ItemSpec, resolver); + newAssemblies.Add (asm); + assemblyPaths.Add (asm, assemblyItem.ItemSpec); + + foreach (MarshalMethodEntry methodEntry in assemblyMarshalMethods) { + TypeDefinition wantedType = methodEntry.NativeCallback.DeclaringType; + TypeDefinition? type = asm.MainModule.FindType (wantedType.FullName); + if (type == null) { + throw new InvalidOperationException ($"Internal error: type '{wantedType.FullName}' not found in assembly '{assemblyItem.ItemSpec}', a linker error?"); + } + + if (type.MetadataToken != wantedType.MetadataToken) { + throw new InvalidOperationException ($"Internal error: type '{type.FullName}' in assembly '{assemblyItem.ItemSpec}' has a different token ID than the original type"); + } + + FindMatchingMethodInType (methodEntry, type, classifier); + } + } + + void FindMatchingMethodInType (MarshalMethodEntry methodEntry, TypeDefinition type, MarshalMethodsClassifier classifier) + { + string callbackName = methodEntry.NativeCallback.FullName; + + foreach (MethodDefinition typeNativeCallbackMethod in type.Methods) { + if (String.Compare (typeNativeCallbackMethod.FullName, callbackName, StringComparison.Ordinal) != 0) { + continue; + } + + if (typeNativeCallbackMethod.Parameters.Count != methodEntry.NativeCallback.Parameters.Count) { + continue; + } + + if (typeNativeCallbackMethod.MetadataToken != methodEntry.NativeCallback.MetadataToken) { + throw new InvalidOperationException ($"Internal error: tokens don't match for '{typeNativeCallbackMethod.FullName}'"); + } + + bool allMatch = true; + for (int i = 0; i < typeNativeCallbackMethod.Parameters.Count; i++) { + if (String.Compare (typeNativeCallbackMethod.Parameters[i].ParameterType.FullName, methodEntry.NativeCallback.Parameters[i].ParameterType.FullName, StringComparison.Ordinal) != 0) { + allMatch = false; + break; + } + } + + if (!allMatch) { + continue; + } + + Log.LogDebugMessage ($"Found match for '{typeNativeCallbackMethod.FullName}' in {type.Module.FileName}"); + string methodKey = classifier.GetStoreMethodKey (methodEntry); + classifier.MarshalMethods[methodKey].Add (new MarshalMethodEntry (methodEntry, typeNativeCallbackMethod)); + } + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs index b17f7beafbb..e748cc8155f 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs @@ -1326,10 +1326,10 @@ public void Dispose () using (var builder = CreateApkBuilder (Path.Combine ("temp", TestContext.CurrentContext.Test.Name))) { builder.ThrowOnBuildFailure = false; Assert.IsFalse (builder.Build (proj), "Build should have failed with XA4212."); - StringAssertEx.Contains ($"error XA4", builder.LastBuildOutput, "Error should be XA4212"); + StringAssertEx.Contains ($"error : XA4", builder.LastBuildOutput, "Error should be XA4212"); StringAssertEx.Contains ($"Type `UnnamedProject.MyBadJavaObject` implements `Android.Runtime.IJavaObject`", builder.LastBuildOutput, "Error should mention MyBadJavaObject"); Assert.IsTrue (builder.Build (proj, parameters: new [] { "AndroidErrorOnCustomJavaObject=False" }), "Build should have succeeded."); - StringAssertEx.Contains ($"warning XA4", builder.LastBuildOutput, "warning XA4212"); + StringAssertEx.Contains ($"warning : XA4", builder.LastBuildOutput, "warning XA4212"); } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs index 33700618d72..219fa75914f 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs @@ -26,7 +26,7 @@ public partial class BuildTest2 : BaseTest new object[] { /* isClassic */ false, /* isRelease */ true, - /* marshalMethodsEnabled */ true, + /* marshalMethodsEnabled */ false, }, new object[] { /* isClassic */ false, diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs index 823c2deec46..5a9cabe50a5 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs @@ -1,9 +1,12 @@ using System; using System.IO; using System.Linq; +using System.Text; using Java.Interop.Tools.Cecil; using Mono.Cecil; +using Mono.Cecil.Cil; using Mono.Linker; +using Mono.Tuner; using MonoDroid.Tuner; using NUnit.Framework; using Xamarin.ProjectTools; @@ -514,7 +517,7 @@ public void AndroidUseNegotiateAuthentication ([Values (true, false, null)] bool } [Test] - public void DoNotErrorOnPerArchJavaTypeDuplicates () + public void DoNotErrorOnPerArchJavaTypeDuplicates ([Values(true, false)] bool enableMarshalMethods) { if (!Builder.UseDotNet) Assert.Ignore ("Test only valid on .NET"); @@ -525,13 +528,24 @@ public void DoNotErrorOnPerArchJavaTypeDuplicates () lib.Sources.Add (new BuildItem.Source ("Library1.cs") { TextContent = () => @" namespace Lib1; -public class Library1 : Java.Lang.Object { +public class Library1 : Com.Example.Androidlib.MyRunner { private static bool Is64Bits = IntPtr.Size >= 8; public static bool Is64 () { return Is64Bits; } + + public override void Run () => Console.WriteLine (Is64Bits); }", + }); + lib.Sources.Add (new BuildItem ("AndroidJavaSource", "MyRunner.java") { + Encoding = new UTF8Encoding (encoderShouldEmitUTF8Identifier: false), + TextContent = () => @" +package com.example.androidlib; + +public abstract class MyRunner { + public abstract void run(); +}" }); var proj = new XamarinAndroidApplicationProject { IsRelease = true, ProjectName = "App1" }; proj.References.Add(new BuildItem.ProjectReference (Path.Combine ("..", "Lib1", "Lib1.csproj"), "Lib1")); @@ -539,12 +553,48 @@ public static bool Is64 () { "base.OnCreate (bundle);", "base.OnCreate (bundle);\n" + "if (Lib1.Library1.Is64 ()) Console.WriteLine (\"Hello World!\");"); + proj.SetProperty ("AndroidEnableMarshalMethods", enableMarshalMethods.ToString ()); using var lb = CreateDllBuilder (Path.Combine (path, "Lib1")); using var b = CreateApkBuilder (Path.Combine (path, "App1")); Assert.IsTrue (lb.Build (lib), "build should have succeeded."); Assert.IsTrue (b.Build (proj), "build should have succeeded."); + + var intermediate = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath); + var dll = $"{lib.ProjectName}.dll"; + Assert64Bit ("android-arm", expected64: false); + Assert64Bit ("android-arm64", expected64: true); + Assert64Bit ("android-x86", expected64: false); + Assert64Bit ("android-x64", expected64: true); + + void Assert64Bit(string rid, bool expected64) + { + var assembly = AssemblyDefinition.ReadAssembly (Path.Combine (intermediate, rid, "linked", "shrunk", dll)); + var type = assembly.MainModule.FindType ("Lib1.Library1"); + Assert.NotNull (type, "Should find Lib1.Library1!"); + var cctor = type.GetTypeConstructor (); + Assert.NotNull (type, "Should find Lib1.Library1.cctor!"); + Assert.AreNotEqual (0, cctor.Body.Instructions.Count); + + /* + * IL snippet + * .method private hidebysig specialname rtspecialname static + * void .cctor () cil managed + * { + * // Is64Bits = 4 >= 8; + * IL_0000: ldc.i4 4 + * IL_0005: ldc.i4.8 + * ... + */ + var instruction = cctor.Body.Instructions [0]; + Assert.AreEqual (OpCodes.Ldc_I4, instruction.OpCode); + if (expected64) { + Assert.AreEqual (8, instruction.Operand, $"Expected 64-bit: {expected64}"); + } else { + Assert.AreEqual (4, instruction.Operand, $"Expected 64-bit: {expected64}"); + } + } } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc index 74cdceacd9e..4c814f7c391 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc @@ -2,64 +2,64 @@ "Comment": null, "Entries": { "AndroidManifest.xml": { - "Size": 3032 + "Size": 3036 }, "assemblies/_Microsoft.Android.Resource.Designer.dll": { "Size": 1024 }, "assemblies/Java.Interop.dll": { - "Size": 58703 + "Size": 58990 }, "assemblies/Mono.Android.dll": { - "Size": 86588 + "Size": 88074 }, "assemblies/Mono.Android.Runtime.dll": { - "Size": 5798 + "Size": 5819 }, "assemblies/rc.bin": { "Size": 1235 }, "assemblies/System.Console.dll": { - "Size": 6442 + "Size": 6448 }, "assemblies/System.Linq.dll": { - "Size": 9123 + "Size": 9135 }, "assemblies/System.Private.CoreLib.dll": { - "Size": 536436 + "Size": 537441 }, "assemblies/System.Runtime.dll": { - "Size": 2623 + "Size": 2629 }, "assemblies/System.Runtime.InteropServices.dll": { - "Size": 3752 + "Size": 3768 }, "assemblies/UnnamedProject.dll": { - "Size": 3349 + "Size": 3222 }, "classes.dex": { - "Size": 19748 + "Size": 377064 }, "lib/arm64-v8a/libmono-component-marshal-ilgen.so": { - "Size": 93552 + "Size": 97392 }, "lib/arm64-v8a/libmonodroid.so": { - "Size": 380832 + "Size": 380704 }, "lib/arm64-v8a/libmonosgen-2.0.so": { - "Size": 3160360 + "Size": 3177168 }, "lib/arm64-v8a/libSystem.IO.Compression.Native.so": { "Size": 723560 }, "lib/arm64-v8a/libSystem.Native.so": { - "Size": 94392 + "Size": 94424 }, "lib/arm64-v8a/libSystem.Security.Cryptography.Native.Android.so": { "Size": 154904 }, "lib/arm64-v8a/libxamarin-app.so": { - "Size": 16624 + "Size": 11080 }, "META-INF/BNDLTOOL.RSA": { "Size": 1213 @@ -95,5 +95,5 @@ "Size": 1904 } }, - "PackageSize": 2685258 + "PackageSize": 2771274 } \ No newline at end of file diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc index 426961a5f03..69c4b5976c5 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc @@ -2,148 +2,148 @@ "Comment": null, "Entries": { "AndroidManifest.xml": { - "Size": 3568 + "Size": 3572 }, "assemblies/_Microsoft.Android.Resource.Designer.dll": { "Size": 2102 }, "assemblies/FormsViewGroup.dll": { - "Size": 7313 + "Size": 7112 }, "assemblies/Java.Interop.dll": { - "Size": 66911 + "Size": 66908 }, "assemblies/Mono.Android.dll": { - "Size": 469830 + "Size": 469884 }, "assemblies/Mono.Android.Runtime.dll": { - "Size": 5818 + "Size": 5819 }, "assemblies/mscorlib.dll": { - "Size": 3866 + "Size": 3865 }, "assemblies/netstandard.dll": { - "Size": 5578 + "Size": 5581 }, "assemblies/rc.bin": { "Size": 1235 }, "assemblies/System.Collections.Concurrent.dll": { - "Size": 11561 + "Size": 11557 }, "assemblies/System.Collections.dll": { - "Size": 15445 + "Size": 15444 }, "assemblies/System.Collections.NonGeneric.dll": { "Size": 7501 }, "assemblies/System.ComponentModel.dll": { - "Size": 1974 + "Size": 1976 }, "assemblies/System.ComponentModel.Primitives.dll": { - "Size": 2596 + "Size": 2598 }, "assemblies/System.ComponentModel.TypeConverter.dll": { - "Size": 6083 + "Size": 6085 }, "assemblies/System.Console.dll": { - "Size": 6612 + "Size": 6614 }, "assemblies/System.Core.dll": { - "Size": 1992 + "Size": 1991 }, "assemblies/System.Diagnostics.TraceSource.dll": { - "Size": 6589 + "Size": 6590 }, "assemblies/System.dll": { - "Size": 2347 + "Size": 2348 }, "assemblies/System.Drawing.dll": { - "Size": 1938 + "Size": 1940 }, "assemblies/System.Drawing.Primitives.dll": { - "Size": 12004 + "Size": 12010 }, "assemblies/System.IO.Compression.Brotli.dll": { - "Size": 11221 + "Size": 11223 }, "assemblies/System.IO.Compression.dll": { - "Size": 15897 + "Size": 15904 }, "assemblies/System.IO.IsolatedStorage.dll": { - "Size": 9913 + "Size": 9912 }, "assemblies/System.Linq.dll": { - "Size": 19490 + "Size": 19495 }, "assemblies/System.Linq.Expressions.dll": { - "Size": 164335 + "Size": 164340 }, "assemblies/System.Net.Http.dll": { - "Size": 65557 + "Size": 65673 }, "assemblies/System.Net.Primitives.dll": { - "Size": 22482 + "Size": 22474 }, "assemblies/System.Net.Requests.dll": { "Size": 3632 }, "assemblies/System.ObjectModel.dll": { - "Size": 8159 + "Size": 8157 }, "assemblies/System.Private.CoreLib.dll": { - "Size": 834340 + "Size": 834482 }, "assemblies/System.Private.DataContractSerialization.dll": { - "Size": 192404 + "Size": 192929 }, "assemblies/System.Private.Uri.dll": { - "Size": 42947 + "Size": 43458 }, "assemblies/System.Private.Xml.dll": { - "Size": 215908 + "Size": 215826 }, "assemblies/System.Private.Xml.Linq.dll": { - "Size": 16681 + "Size": 16684 }, "assemblies/System.Runtime.dll": { - "Size": 2775 + "Size": 2776 }, "assemblies/System.Runtime.InteropServices.dll": { "Size": 3768 }, "assemblies/System.Runtime.Serialization.dll": { - "Size": 1867 + "Size": 1868 }, "assemblies/System.Runtime.Serialization.Formatters.dll": { - "Size": 2518 + "Size": 2520 }, "assemblies/System.Runtime.Serialization.Primitives.dll": { - "Size": 3802 + "Size": 3805 }, "assemblies/System.Security.Cryptography.dll": { - "Size": 8065 + "Size": 8133 }, "assemblies/System.Text.RegularExpressions.dll": { - "Size": 158997 + "Size": 159004 }, "assemblies/System.Xml.dll": { - "Size": 1760 + "Size": 1761 }, "assemblies/System.Xml.Linq.dll": { "Size": 1778 }, "assemblies/UnnamedProject.dll": { - "Size": 5290 + "Size": 5300 }, "assemblies/Xamarin.AndroidX.Activity.dll": { "Size": 5942 }, "assemblies/Xamarin.AndroidX.AppCompat.AppCompatResources.dll": { - "Size": 6261 + "Size": 6033 }, "assemblies/Xamarin.AndroidX.AppCompat.dll": { - "Size": 120195 + "Size": 119847 }, "assemblies/Xamarin.AndroidX.CardView.dll": { "Size": 6799 @@ -152,13 +152,13 @@ "Size": 17257 }, "assemblies/Xamarin.AndroidX.Core.dll": { - "Size": 100933 + "Size": 100666 }, "assemblies/Xamarin.AndroidX.DrawerLayout.dll": { - "Size": 14800 + "Size": 14631 }, "assemblies/Xamarin.AndroidX.Fragment.dll": { - "Size": 41993 + "Size": 41733 }, "assemblies/Xamarin.AndroidX.Legacy.Support.Core.UI.dll": { "Size": 6080 @@ -176,16 +176,16 @@ "Size": 12923 }, "assemblies/Xamarin.AndroidX.RecyclerView.dll": { - "Size": 90383 + "Size": 89997 }, "assemblies/Xamarin.AndroidX.SavedState.dll": { "Size": 4906 }, "assemblies/Xamarin.AndroidX.SwipeRefreshLayout.dll": { - "Size": 10781 + "Size": 10572 }, "assemblies/Xamarin.AndroidX.ViewPager.dll": { - "Size": 18877 + "Size": 18593 }, "assemblies/Xamarin.Forms.Core.dll": { "Size": 528450 @@ -200,31 +200,31 @@ "Size": 60774 }, "assemblies/Xamarin.Google.Android.Material.dll": { - "Size": 42522 + "Size": 42282 }, "classes.dex": { - "Size": 3117140 + "Size": 3514720 }, "lib/arm64-v8a/libmono-component-marshal-ilgen.so": { - "Size": 93552 + "Size": 97392 }, "lib/arm64-v8a/libmonodroid.so": { "Size": 380704 }, "lib/arm64-v8a/libmonosgen-2.0.so": { - "Size": 3169800 + "Size": 3177168 }, "lib/arm64-v8a/libSystem.IO.Compression.Native.so": { "Size": 723560 }, "lib/arm64-v8a/libSystem.Native.so": { - "Size": 94392 + "Size": 94424 }, "lib/arm64-v8a/libSystem.Security.Cryptography.Native.Android.so": { "Size": 154904 }, "lib/arm64-v8a/libxamarin-app.so": { - "Size": 333744 + "Size": 102136 }, "META-INF/android.support.design_material.version": { "Size": 12 @@ -1913,5 +1913,5 @@ "Size": 325240 } }, - "PackageSize": 7900078 + "PackageSize": 7953326 } \ No newline at end of file diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs index 05de2bd0c86..8a1b8996794 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs @@ -83,7 +83,7 @@ internal class ManifestDocument public string ApplicationLabel { get; set; } public string [] Placeholders { get; set; } public List Assemblies { get; set; } - public DirectoryAssemblyResolver Resolver { get; set; } + public IAssemblyResolver Resolver { get; set; } public string SdkDir { get; set; } public string TargetSdkVersion { get; set; } public string MinSdkVersion { get; set; } @@ -255,7 +255,7 @@ void ReorderActivityAliases (TaskLoggingHelper log, XElement app) } } - public IList Merge (TaskLoggingHelper log, TypeDefinitionCache cache, List subclasses, string applicationClass, bool embed, string bundledWearApplicationName, IEnumerable mergedManifestDocuments) + public IList Merge (TaskLoggingHelper log, TypeDefinitionCache cache, List subclasses, string applicationClass, bool embed, string bundledWearApplicationName, IEnumerable mergedManifestDocuments) { var manifest = doc.Root; @@ -330,7 +330,8 @@ public IList Merge (TaskLoggingHelper log, TypeDefinitionCache cache, Li throw new InvalidOperationException (string.Format ("The targetSdkVersion ({0}) is not a valid API level", targetSdkVersion)); int targetSdkVersionValue = tryTargetSdkVersion.Value; - foreach (var t in subclasses) { + foreach (JavaType jt in subclasses) { + TypeDefinition t = jt.Type; if (t.IsAbstract) continue; @@ -567,7 +568,7 @@ Func GetGenerator (T return null; } - XElement CreateApplicationElement (XElement manifest, string applicationClass, List subclasses, TypeDefinitionCache cache) + XElement CreateApplicationElement (XElement manifest, string applicationClass, List subclasses, TypeDefinitionCache cache) { var application = manifest.Descendants ("application").FirstOrDefault (); @@ -591,7 +592,8 @@ XElement CreateApplicationElement (XElement manifest, string applicationClass, L List typeAttr = new List (); List typeUsesLibraryAttr = new List (); List typeUsesConfigurationAttr = new List (); - foreach (var t in subclasses) { + foreach (JavaType jt in subclasses) { + TypeDefinition t = jt.Type; ApplicationAttribute aa = ApplicationAttribute.FromCustomAttributeProvider (t); if (aa == null) continue; @@ -923,7 +925,7 @@ void AddSupportsGLTextures (XElement application, TypeDefinitionCache cache) } } - void AddInstrumentations (XElement manifest, IList subclasses, int targetSdkVersion, TypeDefinitionCache cache) + void AddInstrumentations (XElement manifest, IList subclasses, int targetSdkVersion, TypeDefinitionCache cache) { var assemblyAttrs = Assemblies.SelectMany (path => InstrumentationAttribute.FromCustomAttributeProvider (Resolver.GetAssembly (path))); @@ -936,12 +938,14 @@ void AddInstrumentations (XElement manifest, IList subclasses, i manifest.Add (ia.ToElement (PackageName, cache)); } - foreach (var type in subclasses) + foreach (JavaType jt in subclasses) { + TypeDefinition type = jt.Type; if (type.IsSubclassOf ("Android.App.Instrumentation", cache)) { var xe = InstrumentationFromTypeDefinition (type, JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'), cache); if (xe != null) manifest.Add (xe); } + } } public bool SaveIfChanged (TaskLoggingHelper log, string filename) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs index 4131349d150..96063bf2126 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs @@ -23,32 +23,24 @@ sealed class AssemblyImports IDictionary> methods; ICollection uniqueAssemblies; - IDictionary > assemblyPaths; + IDictionary assemblyPaths; TaskLoggingHelper log; - public MarshalMethodsAssemblyRewriter (IDictionary> methods, ICollection uniqueAssemblies, IDictionary > assemblyPaths, TaskLoggingHelper log) + public MarshalMethodsAssemblyRewriter (IDictionary> methods, ICollection uniqueAssemblies, IDictionary assemblyPaths, TaskLoggingHelper log) { + this.assemblyPaths = assemblyPaths; this.methods = methods ?? throw new ArgumentNullException (nameof (methods)); this.uniqueAssemblies = uniqueAssemblies ?? throw new ArgumentNullException (nameof (uniqueAssemblies)); - this.assemblyPaths = assemblyPaths ?? throw new ArgumentNullException (nameof (assemblyPaths)); this.log = log ?? throw new ArgumentNullException (nameof (log)); } // TODO: do away with broken exception transitions, there's no point in supporting them - public void Rewrite (DirectoryAssemblyResolver resolver, List targetAssemblyPaths, bool brokenExceptionTransitions) + public void Rewrite (XAAssemblyResolver resolver, bool brokenExceptionTransitions) { if (resolver == null) { throw new ArgumentNullException (nameof (resolver)); } - if (targetAssemblyPaths == null) { - throw new ArgumentNullException (nameof (targetAssemblyPaths)); - } - - if (targetAssemblyPaths.Count == 0) { - throw new ArgumentException ("must contain at least one target path", nameof (targetAssemblyPaths)); - } - AssemblyDefinition? monoAndroidRuntime = resolver.Resolve ("Mono.Android.Runtime"); if (monoAndroidRuntime == null) { throw new InvalidOperationException ($"Internal error: unable to load the Mono.Android.Runtime assembly"); @@ -114,53 +106,58 @@ public void Rewrite (DirectoryAssemblyResolver resolver, List targetAsse } } - var newAssemblyPaths = new List (); foreach (AssemblyDefinition asm in uniqueAssemblies) { - foreach (string path in GetAssemblyPaths (asm)) { - var writerParams = new WriterParameters { - WriteSymbols = File.Exists (Path.ChangeExtension (path, ".pdb")), - }; - - string directory = Path.Combine (Path.GetDirectoryName (path), "new"); - Directory.CreateDirectory (directory); - string output = Path.Combine (directory, Path.GetFileName (path)); - log.LogDebugMessage ($"Writing new version of assembly: {output}"); - - // TODO: this should be used eventually, but it requires that all the types are reloaded from the assemblies before typemaps are generated - // since Cecil doesn't update the MVID in the already loaded types - //asm.MainModule.Mvid = Guid.NewGuid (); - asm.Write (output, writerParams); - newAssemblyPaths.Add (output); - } - } + string path = GetAssemblyPath (asm); + string pathPdb = Path.ChangeExtension (path, ".pdb"); + bool havePdb = File.Exists (pathPdb); - // Replace old versions of the assemblies only after we've finished rewriting without issues, otherwise leave the new - // versions around. - foreach (string path in newAssemblyPaths) { - string? pdb = null; + var writerParams = new WriterParameters { + WriteSymbols = havePdb, + }; - string source = Path.ChangeExtension (path, ".pdb"); - if (File.Exists (source)) { - pdb = source; - } + string directory = Path.Combine (Path.GetDirectoryName (path), "new"); + Directory.CreateDirectory (directory); + string output = Path.Combine (directory, Path.GetFileName (path)); + log.LogDebugMessage ($"Writing new version of '{path}' assembly: {output}"); - foreach (string targetPath in targetAssemblyPaths) { - string target = Path.Combine (targetPath, Path.GetFileName (path)); - CopyFile (path, target); + // TODO: this should be used eventually, but it requires that all the types are reloaded from the assemblies before typemaps are generated + // since Cecil doesn't update the MVID in the already loaded types + //asm.MainModule.Mvid = Guid.NewGuid (); + asm.Write (output, writerParams); - if (!String.IsNullOrEmpty (pdb)) { - CopyFile (pdb, Path.ChangeExtension (target, ".pdb")); + CopyFile (output, path); + RemoveFile (output); + + if (havePdb) { + string outputPdb = Path.ChangeExtension (output, ".pdb"); + if (File.Exists (outputPdb)) { + CopyFile (outputPdb, pathPdb); } + RemoveFile (outputPdb); } - - RemoveFile (path); - RemoveFile (pdb); } void CopyFile (string source, string target) { log.LogDebugMessage ($"Copying rewritten assembly: {source} -> {target}"); + + string targetBackup = $"{target}.bak"; + if (File.Exists (target)) { + // Try to avoid sharing violations by first renaming the target + File.Move (target, targetBackup); + } + File.Copy (source, target, true); + + if (File.Exists (targetBackup)) { + try { + File.Delete (targetBackup); + } catch (Exception ex) { + // On Windows the deletion may fail, depending on lock state of the original `target` file before the move. + log.LogDebugMessage ($"While trying to delete '{targetBackup}', exception was thrown: {ex}"); + log.LogDebugMessage ($"Failed to delete backup file '{targetBackup}', ignoring."); + } + } } void RemoveFile (string? path) @@ -452,16 +449,18 @@ TypeReference ReturnValid (Type typeToLookUp) } } - ICollection GetAssemblyPaths (AssemblyDefinition asm) + string GetAssemblyPath (AssemblyDefinition asm) { - if (!assemblyPaths.TryGetValue (asm.Name.Name, out HashSet paths)) { - throw new InvalidOperationException ($"Unable to determine file path for assembly '{asm.Name.Name}'"); + string filePath = asm.MainModule.FileName; + if (!String.IsNullOrEmpty (filePath)) { + return filePath; } - return paths; + // No checking on purpose - the assembly **must** be there if its MainModule.FileName property returns a null or empty string + return assemblyPaths[asm]; } - MethodDefinition GetUnmanagedCallersOnlyAttributeConstructor (DirectoryAssemblyResolver resolver) + MethodDefinition GetUnmanagedCallersOnlyAttributeConstructor (XAAssemblyResolver resolver) { AssemblyDefinition asm = resolver.Resolve ("System.Runtime.InteropServices"); TypeDefinition unmanagedCallersOnlyAttribute = null; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs index 3a7d4712b3f..e7418e430a1 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs @@ -40,8 +40,8 @@ sealed class MarshalMethodEntry public bool IsSpecial { get; } public MarshalMethodEntry (TypeDefinition declaringType, MethodDefinition nativeCallback, MethodDefinition connector, MethodDefinition - registeredMethod, MethodDefinition implementedMethod, FieldDefinition callbackField, string jniTypeName, - string jniName, string jniSignature, bool needsBlittableWorkaround) + registeredMethod, MethodDefinition implementedMethod, FieldDefinition callbackField, string jniTypeName, + string jniName, string jniSignature, bool needsBlittableWorkaround) { DeclaringType = declaringType ?? throw new ArgumentNullException (nameof (declaringType)); nativeCallbackReal = nativeCallback ?? throw new ArgumentNullException (nameof (nativeCallback)); @@ -66,6 +66,12 @@ public MarshalMethodEntry (TypeDefinition declaringType, MethodDefinition native IsSpecial = true; } + public MarshalMethodEntry (MarshalMethodEntry other, MethodDefinition nativeCallback) + : this (other.DeclaringType, nativeCallback, other.Connector, other.RegisteredMethod, + other.ImplementedMethod, other.CallbackField, other.JniTypeName, other.JniMethodName, + other.JniMethodSignature, other.NeedsBlittableWorkaround) + {} + string EnsureNonEmpty (string s, string argName) { if (String.IsNullOrEmpty (s)) { @@ -218,7 +224,7 @@ public bool Matches (MethodDefinition method) } TypeDefinitionCache tdCache; - DirectoryAssemblyResolver resolver; + XAAssemblyResolver resolver; Dictionary> marshalMethods; HashSet assemblies; TaskLoggingHelper log; @@ -231,7 +237,7 @@ public bool Matches (MethodDefinition method) public ulong RejectedMethodCount => rejectedMethodCount; public ulong WrappedMethodCount => wrappedMethodCount; - public MarshalMethodsClassifier (TypeDefinitionCache tdCache, DirectoryAssemblyResolver res, TaskLoggingHelper log) + public MarshalMethodsClassifier (TypeDefinitionCache tdCache, XAAssemblyResolver res, TaskLoggingHelper log) { this.log = log ?? throw new ArgumentNullException (nameof (log)); this.tdCache = tdCache ?? throw new ArgumentNullException (nameof (tdCache)); @@ -499,7 +505,6 @@ bool IsStandardHandler (TypeDefinition topType, ConnectorInfo connector, MethodD // method.CallbackField?.DeclaringType.Fields == 'null' StoreMethod ( - registeredMethod, new MarshalMethodEntry ( topType, nativeCallbackMethod, @@ -683,10 +688,16 @@ FieldDefinition FindField (TypeDefinition type, string fieldName, bool lookForIn return FindField (tdCache.Resolve (type.BaseType), fieldName, lookForInherited); } - void StoreMethod (MethodDefinition registeredMethod, MarshalMethodEntry entry) + public string GetStoreMethodKey (MarshalMethodEntry methodEntry) { + MethodDefinition registeredMethod = methodEntry.RegisteredMethod; string typeName = registeredMethod.DeclaringType.FullName.Replace ('/', '+'); - string key = $"{typeName}, {registeredMethod.DeclaringType.GetPartialAssemblyName (tdCache)}\t{registeredMethod.Name}"; + return $"{typeName}, {registeredMethod.DeclaringType.GetPartialAssemblyName (tdCache)}\t{registeredMethod.Name}"; + } + + void StoreMethod (MarshalMethodEntry entry) + { + string key = GetStoreMethodKey (entry); // Several classes can override the same method, we need to generate the marshal method only once, at the same time // keeping track of overloads @@ -706,7 +717,6 @@ void StoreAssembly (AssemblyDefinition asm) if (assemblies.Contains (asm)) { return; } - assemblies.Add (asm); } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index 8579f37a8c0..7e27360dd96 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -536,6 +536,17 @@ public static string GetRelativePathForAndroidAsset (string assetsDirectory, ITa return path; } + public static AndroidTargetArch AbiToTargetArch (string abi) + { + return abi switch { + "armeabi-v7a" => AndroidTargetArch.Arm, + "arm64-v8a" => AndroidTargetArch.Arm64, + "x86_64" => AndroidTargetArch.X86_64, + "x86" => AndroidTargetArch.X86, + _ => throw new NotSupportedException ($"Internal error: unsupported ABI '{abi}'") + }; + } + public static string? CultureInvariantToString (object? obj) { if (obj == null) { @@ -561,5 +572,15 @@ public static int ConvertSupportedOSPlatformVersionToApiLevel (string version) } return apiLevel; } + + public static AndroidTargetArch GetTargetArch (ITaskItem asmItem) + { + string? abi = asmItem.GetMetadata ("Abi"); + if (String.IsNullOrEmpty (abi)) { + return AndroidTargetArch.None; + } + + return AbiToTargetArch (abi); + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs index f59656518fc..1b020083282 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.Collections.Generic; +using System.Collections.ObjectModel; using System.Linq; using System.Text; @@ -89,6 +90,49 @@ internal sealed class ModuleDebugData public byte[] ModuleNameBytes; } + sealed class ReleaseGenerationState + { + int assemblyId = 0; + + public readonly Dictionary KnownAssemblies; + public readonly Dictionary MvidCache; + public readonly IDictionary> TempModules; + + // Just a convenient way to access one of the temp modules dictionaries, to be used when dealing with ABI-agnostic + // types in ProcessReleaseType. + public readonly Dictionary TempModulesAbiAgnostic; + + public ReleaseGenerationState (string[] supportedAbis) + { + KnownAssemblies = new Dictionary (StringComparer.Ordinal); + MvidCache = new Dictionary (); + + var tempModules = new Dictionary> (); + foreach (string abi in supportedAbis) { + var dict = new Dictionary (); + if (TempModulesAbiAgnostic == null) { + TempModulesAbiAgnostic = dict; + } + tempModules.Add (AbiToArch (abi), dict); + } + + TempModules = new ReadOnlyDictionary> (tempModules); + } + + public void AddKnownAssembly (TypeDefinition td) + { + string assemblyName = GetAssemblyName (td); + + if (KnownAssemblies.ContainsKey (assemblyName)) { + return; + } + + KnownAssemblies.Add (assemblyName, ++assemblyId); + } + + public string GetAssemblyName (TypeDefinition td) => td.Module.Assembly.FullName; + } + Action logger; Encoding outputEncoding; byte[] moduleMagicString; @@ -124,7 +168,7 @@ void UpdateApplicationConfig (TypeDefinition javaType, ApplicationConfigTaskStat } } - public bool Generate (bool debugBuild, bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, bool generateNativeAssembly, out ApplicationConfigTaskState appConfState) + public bool Generate (bool debugBuild, bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, bool generateNativeAssembly, out ApplicationConfigTaskState appConfState) { if (String.IsNullOrEmpty (outputDirectory)) throw new ArgumentException ("must not be null or empty", nameof (outputDirectory)); @@ -145,21 +189,23 @@ public bool Generate (bool debugBuild, bool skipJniAddNativeMethodRegistrationAt return GenerateRelease (skipJniAddNativeMethodRegistrationAttributeScan, javaTypes, cache, typemapsOutputDirectory, appConfState); } - bool GenerateDebug (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, bool generateNativeAssembly, ApplicationConfigTaskState appConfState) + bool GenerateDebug (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, bool generateNativeAssembly, ApplicationConfigTaskState appConfState) { - if (generateNativeAssembly) + if (generateNativeAssembly) { return GenerateDebugNativeAssembly (skipJniAddNativeMethodRegistrationAttributeScan, javaTypes, cache, outputDirectory, appConfState); + } return GenerateDebugFiles (skipJniAddNativeMethodRegistrationAttributeScan, javaTypes, cache, outputDirectory, appConfState); } - bool GenerateDebugFiles (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) + bool GenerateDebugFiles (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) { var modules = new Dictionary (StringComparer.Ordinal); int maxModuleFileNameWidth = 0; int maxModuleNameWidth = 0; var javaDuplicates = new Dictionary> (StringComparer.Ordinal); - foreach (TypeDefinition td in javaTypes) { + foreach (JavaType jt in javaTypes) { + TypeDefinition td = jt.Type; UpdateApplicationConfig (td, appConfState); string moduleName = td.Module.Assembly.Name.Name; ModuleDebugData module; @@ -218,13 +264,14 @@ bool GenerateDebugFiles (bool skipJniAddNativeMethodRegistrationAttributeScan, L return true; } - bool GenerateDebugNativeAssembly (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) + bool GenerateDebugNativeAssembly (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) { var javaToManaged = new List (); var managedToJava = new List (); var javaDuplicates = new Dictionary> (StringComparer.Ordinal); - foreach (TypeDefinition td in javaTypes) { + foreach (JavaType jt in javaTypes) { + TypeDefinition td = jt.Type; UpdateApplicationConfig (td, appConfState); TypeMapDebugEntry entry = GetDebugEntry (td, cache); @@ -330,91 +377,118 @@ string GetManagedTypeName (TypeDefinition td) return $"{managedTypeName}, {td.Module.Assembly.Name.Name}"; } - bool GenerateRelease (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) + void ProcessReleaseType (ReleaseGenerationState state, TypeDefinition td, AndroidTargetArch typeArch, ApplicationConfigTaskState appConfState, TypeDefinitionCache cache) { - int assemblyId = 0; - var knownAssemblies = new Dictionary (StringComparer.Ordinal); - var tempModules = new Dictionary (); - Dictionary moduleCounter = null; - var mvidCache = new Dictionary (); + UpdateApplicationConfig (td, appConfState); - foreach (TypeDefinition td in javaTypes) { - UpdateApplicationConfig (td, appConfState); + state.AddKnownAssembly (td); - string assemblyName = td.Module.Assembly.FullName; + // We must NOT use Guid here! The reason is that Guid sort order is different than its corresponding + // byte array representation and on the runtime we need the latter in order to be able to binary search + // through the module array. + byte[] moduleUUID; + if (!state.MvidCache.TryGetValue (td.Module.Mvid, out moduleUUID)) { + moduleUUID = td.Module.Mvid.ToByteArray (); + state.MvidCache.Add (td.Module.Mvid, moduleUUID); + } - if (!knownAssemblies.ContainsKey (assemblyName)) { - assemblyId++; - knownAssemblies.Add (assemblyName, assemblyId); + bool abiAgnosticType = typeArch == AndroidTargetArch.None; + Dictionary tempModules; + if (abiAgnosticType) { + tempModules = state.TempModulesAbiAgnostic; + } else { + // It will throw if `typeArch` isn't in the dictionary. This is intentional, since we must have no TypeDefinition entries for architectures not + // mentioned in `supportedAbis`. + try { + tempModules = state.TempModules[typeArch]; + } catch (KeyNotFoundException ex) { + throw new InvalidOperationException ($"Internal error: cannot process type specific to architecture '{typeArch}', since that architecture isn't mentioned in the set of supported ABIs", ex); } + } - // We must NOT use Guid here! The reason is that Guid sort order is different than its corresponding - // byte array representation and on the runtime we need the latter in order to be able to binary search - // through the module array. - byte[] moduleUUID; - if (!mvidCache.TryGetValue (td.Module.Mvid, out moduleUUID)) { - moduleUUID = td.Module.Mvid.ToByteArray (); - mvidCache.Add (td.Module.Mvid, moduleUUID); - } + if (!tempModules.TryGetValue (moduleUUID, out ModuleReleaseData moduleData)) { + moduleData = new ModuleReleaseData { + Mvid = td.Module.Mvid, + MvidBytes = moduleUUID, + Assembly = td.Module.Assembly, + AssemblyName = td.Module.Assembly.Name.Name, + TypesScratch = new Dictionary (StringComparer.Ordinal), + DuplicateTypes = new List (), + }; - ModuleReleaseData moduleData; - if (!tempModules.TryGetValue (moduleUUID, out moduleData)) { - if (moduleCounter == null) - moduleCounter = new Dictionary (); - - moduleData = new ModuleReleaseData { - Mvid = td.Module.Mvid, - MvidBytes = moduleUUID, - Assembly = td.Module.Assembly, - AssemblyName = td.Module.Assembly.Name.Name, - TypesScratch = new Dictionary (StringComparer.Ordinal), - DuplicateTypes = new List (), - }; + if (abiAgnosticType) { + // ABI-agnostic types must be added to all the ABIs + foreach (var kvp in state.TempModules) { + kvp.Value.Add (moduleUUID, moduleData); + } + } else { + // ABI-specific types are added only to their respective tempModules tempModules.Add (moduleUUID, moduleData); } + } - string javaName = Java.Interop.Tools.TypeNameMappings.JavaNativeTypeManager.ToJniName (td, cache); - // We will ignore generic types and interfaces when generating the Java to Managed map, but we must not - // omit them from the table we output - we need the same number of entries in both java-to-managed and - // managed-to-java tables. `SkipInJavaToManaged` set to `true` will cause the native assembly generator - // to output `0` as the token id for the type, thus effectively causing the runtime unable to match such - // a Java type name to a managed type. This fixes https://github.com/xamarin/xamarin-android/issues/4660 - var entry = new TypeMapReleaseEntry { - JavaName = javaName, - ManagedTypeName = td.FullName, - Token = td.MetadataToken.ToUInt32 (), - AssemblyNameIndex = knownAssemblies [assemblyName], - SkipInJavaToManaged = ShouldSkipInJavaToManaged (td), - }; + string javaName = Java.Interop.Tools.TypeNameMappings.JavaNativeTypeManager.ToJniName (td, cache); + // We will ignore generic types and interfaces when generating the Java to Managed map, but we must not + // omit them from the table we output - we need the same number of entries in both java-to-managed and + // managed-to-java tables. `SkipInJavaToManaged` set to `true` will cause the native assembly generator + // to output `0` as the token id for the type, thus effectively causing the runtime unable to match such + // a Java type name to a managed type. This fixes https://github.com/xamarin/xamarin-android/issues/4660 + var entry = new TypeMapReleaseEntry { + JavaName = javaName, + ManagedTypeName = td.FullName, + Token = td.MetadataToken.ToUInt32 (), + AssemblyNameIndex = state.KnownAssemblies [state.GetAssemblyName (td)], + SkipInJavaToManaged = ShouldSkipInJavaToManaged (td), + }; - if (moduleData.TypesScratch.ContainsKey (entry.JavaName)) { - // This is disabled because it costs a lot of time (around 150ms per standard XF Integration app - // build) and has no value for the end user. The message is left here because it may be useful to us - // in our devloop at some point. - //logger ($"Warning: duplicate Java type name '{entry.JavaName}' in assembly '{moduleData.AssemblyName}' (new token: {entry.Token})."); - moduleData.DuplicateTypes.Add (entry); - } else - moduleData.TypesScratch.Add (entry.JavaName, entry); + if (moduleData.TypesScratch.ContainsKey (entry.JavaName)) { + // This is disabled because it costs a lot of time (around 150ms per standard XF Integration app + // build) and has no value for the end user. The message is left here because it may be useful to us + // in our devloop at some point. + //logger ($"Warning: duplicate Java type name '{entry.JavaName}' in assembly '{moduleData.AssemblyName}' (new token: {entry.Token})."); + moduleData.DuplicateTypes.Add (entry); + } else { + moduleData.TypesScratch.Add (entry.JavaName, entry); } + } - var modules = tempModules.Values.ToArray (); - Array.Sort (modules, new ModuleUUIDArrayComparer ()); + bool GenerateRelease (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) + { + var state = new ReleaseGenerationState (supportedAbis); - foreach (ModuleReleaseData module in modules) { - if (module.TypesScratch.Count == 0) { - module.Types = Array.Empty (); + foreach (JavaType jt in javaTypes) { + if (!jt.IsABiSpecific) { + ProcessReleaseType (state, jt.Type, AndroidTargetArch.None, appConfState, cache); continue; } - // No need to sort here, the LLVM IR generator will compute hashes and sort - // the array on write. - module.Types = module.TypesScratch.Values.ToArray (); + foreach (var kvp in jt.PerAbiTypes) { + ProcessReleaseType (state, kvp.Value, kvp.Key, appConfState, cache); + } } - NativeTypeMappingData data; - data = new NativeTypeMappingData (logger, modules); + var mappingData = new Dictionary (); + foreach (var kvp in state.TempModules) { + AndroidTargetArch arch = kvp.Key; + Dictionary tempModules = kvp.Value; + var modules = tempModules.Values.ToArray (); + Array.Sort (modules, new ModuleUUIDArrayComparer ()); + + foreach (ModuleReleaseData module in modules) { + if (module.TypesScratch.Count == 0) { + module.Types = Array.Empty (); + continue; + } + + // No need to sort here, the LLVM IR generator will compute hashes and sort + // the array on write. + module.Types = module.TypesScratch.Values.ToArray (); + } + + mappingData.Add (arch, new NativeTypeMappingData (logger, modules)); + } - var generator = new TypeMappingReleaseNativeAssemblyGenerator (data); + var generator = new TypeMappingReleaseNativeAssemblyGenerator (mappingData); generator.Init (); GenerateNativeAssembly (generator, outputDirectory); @@ -430,27 +504,7 @@ void GenerateNativeAssembly (TypeMappingAssemblyGenerator generator, string base { AndroidTargetArch arch; foreach (string abi in supportedAbis) { - switch (abi.Trim ()) { - case "armeabi-v7a": - arch = AndroidTargetArch.Arm; - break; - - case "arm64-v8a": - arch = AndroidTargetArch.Arm64; - break; - - case "x86": - arch = AndroidTargetArch.X86; - break; - - case "x86_64": - arch = AndroidTargetArch.X86_64; - break; - - default: - throw new InvalidOperationException ($"Unknown ABI {abi}"); - } - + arch = AbiToArch (abi); string outputFile = $"{baseFileName}.{abi}.ll"; using (var sw = MemoryStreamPool.Shared.CreateStreamWriter (outputEncoding)) { generator.Write (arch, sw, outputFile); @@ -460,6 +514,17 @@ void GenerateNativeAssembly (TypeMappingAssemblyGenerator generator, string base } } + static AndroidTargetArch AbiToArch (string abi) + { + return abi switch { + "armeabi-v7a" => AndroidTargetArch.Arm, + "arm64-v8a" => AndroidTargetArch.Arm64, + "x86_64" => AndroidTargetArch.X86_64, + "x86" => AndroidTargetArch.X86, + _ => throw new InvalidOperationException ($"Unknown ABI {abi}") + }; + } + // Binary index file format, all data is little-endian: // // [Magic string] # XATI diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs index bc407b478aa..f92af2e902c 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs @@ -5,6 +5,7 @@ using System.Text; using Xamarin.Android.Tasks.LLVMIR; +using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks { @@ -144,56 +145,80 @@ public ModuleMapData (string symbolLabel, List> MapModules; + public readonly List> JavaMap; + public readonly Dictionary JavaTypesByName; + public readonly List JavaNames; + public readonly NativeTypeMappingData MappingData; + public ulong ModuleCounter = 0; + + public ArchGenerationState (NativeTypeMappingData mappingData) + { + MapModules = new List> (); + JavaMap = new List> (); + JavaTypesByName = new Dictionary (StringComparer.Ordinal); + JavaNames = new List (); + MappingData = mappingData; + } + } + StructureInfo typeMapJavaStructureInfo; StructureInfo typeMapModuleStructureInfo; StructureInfo typeMapModuleEntryStructureInfo; - List> mapModules; - List> javaMap; - Dictionary javaTypesByName; - List javaNames; - JavaNameHashComparer javaNameHashComparer; + Dictionary archState; - ulong moduleCounter = 0; + JavaNameHashComparer javaNameHashComparer; - public TypeMappingReleaseNativeAssemblyGenerator (NativeTypeMappingData mappingData) + public TypeMappingReleaseNativeAssemblyGenerator (Dictionary mappingData) { - this.mappingData = mappingData ?? throw new ArgumentNullException (nameof (mappingData)); - mapModules = new List> (); - javaMap = new List> (); - javaTypesByName = new Dictionary (StringComparer.Ordinal); + if (mappingData == null) { + throw new ArgumentNullException (nameof (mappingData)); + } + javaNameHashComparer = new JavaNameHashComparer (); - javaNames = new List (); + archState = new Dictionary (mappingData.Count); + + foreach (var kvp in mappingData) { + if (kvp.Value == null) { + throw new ArgumentException ("must not contain null values", nameof (mappingData)); + } + + archState.Add (kvp.Key, new ArchGenerationState (kvp.Value)); + } } public override void Init () { - InitMapModules (); - InitJavaMap (); + foreach (var kvp in archState) { + InitMapModules (kvp.Value); + InitJavaMap (kvp.Value); + } } - void InitJavaMap () + void InitJavaMap (ArchGenerationState state) { TypeMapJava map_entry; - foreach (TypeMapGenerator.TypeMapReleaseEntry entry in mappingData.JavaTypes) { - javaNames.Add (entry.JavaName); + foreach (TypeMapGenerator.TypeMapReleaseEntry entry in state.MappingData.JavaTypes) { + state.JavaNames.Add (entry.JavaName); map_entry = new TypeMapJava { module_index = (uint)entry.ModuleIndex, // UInt32.MaxValue, type_token_id = entry.SkipInJavaToManaged ? 0 : entry.Token, - java_name_index = (uint)(javaNames.Count - 1), + java_name_index = (uint)(state.JavaNames.Count - 1), JavaName = entry.JavaName, }; - javaMap.Add (new StructureInstance (map_entry)); - javaTypesByName.Add (map_entry.JavaName, map_entry); + state.JavaMap.Add (new StructureInstance (map_entry)); + state.JavaTypesByName.Add (map_entry.JavaName, map_entry); } } - void InitMapModules () + void InitMapModules (ArchGenerationState state) { - foreach (TypeMapGenerator.ModuleReleaseData data in mappingData.Modules) { - string mapName = $"module{moduleCounter++}_managed_to_java"; + foreach (TypeMapGenerator.ModuleReleaseData data in state.MappingData.Modules) { + string mapName = $"module{state.ModuleCounter++}_managed_to_java"; string duplicateMapName; if (data.DuplicateTypes.Count == 0) @@ -214,7 +239,7 @@ void InitMapModules () java_name_width = 0, }; - mapModules.Add (new StructureInstance (map_module)); + state.MapModules.Add (new StructureInstance (map_module)); } } @@ -228,7 +253,7 @@ protected override void MapStructures (LlvmIrGenerator generator) // Prepare module map entries by sorting them on the managed token, and then mapping each entry to its corresponding Java type map index. // Requires that `javaMap` is sorted on the type name hash. - void PrepareMapModuleData (string moduleDataSymbolLabel, IEnumerable moduleEntries, List allModulesData) + void PrepareMapModuleData (ArchGenerationState state, string moduleDataSymbolLabel, IEnumerable moduleEntries, List allModulesData) { var mapModuleEntries = new List> (); foreach (TypeMapGenerator.TypeMapReleaseEntry entry in moduleEntries) { @@ -244,12 +269,12 @@ void PrepareMapModuleData (string moduleDataSymbolLabel, IEnumerable (javaType); - int idx = javaMap.BinarySearch (key, javaNameHashComparer); + int idx = state.JavaMap.BinarySearch (key, javaNameHashComparer); if (idx < 0) { throw new InvalidOperationException ($"Could not map entry '{javaTypeName}' to array index"); } @@ -261,32 +286,32 @@ uint GetJavaEntryIndex (string javaTypeName) // Generate hashes for all Java type names, then sort javaMap on the name hash. This has to be done in the writing phase because hashes // will depend on architecture (or, actually, on its bitness) and may differ between architectures (they will be the same for all architectures // with the same bitness) - (List allMapModulesData, List javaMapHashes) PrepareMapsForWriting (LlvmIrGenerator generator) + (List allMapModulesData, List javaMapHashes) PrepareMapsForWriting (ArchGenerationState state, LlvmIrGenerator generator) { bool is64Bit = generator.Is64Bit; // Generate Java type name hashes... - for (int i = 0; i < javaMap.Count; i++) { - TypeMapJava entry = javaMap[i].Obj; + for (int i = 0; i < state.JavaMap.Count; i++) { + TypeMapJava entry = state.JavaMap[i].Obj; entry.JavaNameHash = HashName (entry.JavaName); } // ...sort them... - javaMap.Sort ((StructureInstance a, StructureInstance b) => a.Obj.JavaNameHash.CompareTo (b.Obj.JavaNameHash)); + state.JavaMap.Sort ((StructureInstance a, StructureInstance b) => a.Obj.JavaNameHash.CompareTo (b.Obj.JavaNameHash)); var allMapModulesData = new List (); // ...and match managed types to Java... - foreach (StructureInstance moduleInstance in mapModules) { + foreach (StructureInstance moduleInstance in state.MapModules) { TypeMapModule module = moduleInstance.Obj; - PrepareMapModuleData (module.MapSymbolName, module.Data.Types, allMapModulesData); + PrepareMapModuleData (state, module.MapSymbolName, module.Data.Types, allMapModulesData); if (module.Data.DuplicateTypes.Count > 0) { - PrepareMapModuleData (module.DuplicateMapSymbolName, module.Data.DuplicateTypes, allMapModulesData); + PrepareMapModuleData (state, module.DuplicateMapSymbolName, module.Data.DuplicateTypes, allMapModulesData); } } var javaMapHashes = new HashSet (); - foreach (StructureInstance entry in javaMap) { + foreach (StructureInstance entry in state.JavaMap) { javaMapHashes.Add (entry.Obj.JavaNameHash); } @@ -315,22 +340,30 @@ ulong HashBytes (byte[] bytes) protected override void Write (LlvmIrGenerator generator) { - generator.WriteVariable ("map_module_count", mappingData.MapModuleCount); - generator.WriteVariable ("java_type_count", javaMap.Count); // must include the padding item, if any + ArchGenerationState state; + + try { + state = archState[generator.TargetArch]; + } catch (KeyNotFoundException ex) { + throw new InvalidOperationException ($"Internal error: architecture {generator.TargetArch} has not been prepared for writing.", ex); + } + + generator.WriteVariable ("map_module_count", state.MappingData.MapModuleCount); + generator.WriteVariable ("java_type_count", state.JavaMap.Count); // must include the padding item, if any - (List allMapModulesData, List javaMapHashes) = PrepareMapsForWriting (generator); - WriteMapModules (generator, allMapModulesData); - WriteJavaMap (generator, javaMapHashes); + (List allMapModulesData, List javaMapHashes) = PrepareMapsForWriting (state, generator); + WriteMapModules (state, generator, allMapModulesData); + WriteJavaMap (state, generator, javaMapHashes); } - void WriteJavaMap (LlvmIrGenerator generator, List javaMapHashes) + void WriteJavaMap (ArchGenerationState state, LlvmIrGenerator generator, List javaMapHashes) { generator.WriteEOL (); generator.WriteEOL ("Java to managed map"); generator.WriteStructureArray ( typeMapJavaStructureInfo, - javaMap, + state.JavaMap, LlvmIrVariableOptions.GlobalConstant, "map_java" ); @@ -347,7 +380,7 @@ void WriteJavaMap (LlvmIrGenerator generator, List javaMapHashes) WriteHashes (hashes); } - generator.WriteArray (javaNames, "java_type_names"); + generator.WriteArray (state.JavaNames, "java_type_names"); void WriteHashes (List hashes) where T: struct { @@ -355,14 +388,14 @@ void WriteHashes (List hashes) where T: struct hashes, LlvmIrVariableOptions.GlobalConstant, "map_java_hashes", - (int idx, T value) => $"{idx}: 0x{value:x} => {javaMap[idx].Obj.JavaName}" + (int idx, T value) => $"{idx}: 0x{value:x} => {state.JavaMap[idx].Obj.JavaName}" ); } } - void WriteMapModules (LlvmIrGenerator generator, List mapModulesData) + void WriteMapModules (ArchGenerationState state, LlvmIrGenerator generator, List mapModulesData) { - if (mapModules.Count == 0) { + if (state.MapModules.Count == 0) { return; } @@ -381,7 +414,7 @@ void WriteMapModules (LlvmIrGenerator generator, List mapModulesD generator.WriteEOL ("Map modules"); generator.WriteStructureArray ( typeMapModuleStructureInfo, - mapModules, + state.MapModules, LlvmIrVariableOptions.GlobalWritable, "map_modules" ); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs b/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs new file mode 100644 index 00000000000..746f45802e7 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs @@ -0,0 +1,326 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.MemoryMappedFiles; + +using Microsoft.Build.Utilities; +using Mono.Cecil; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +class XAAssemblyResolver : IAssemblyResolver +{ + sealed class CacheEntry : IDisposable + { + bool disposed; + + Dictionary assemblies; + TaskLoggingHelper log; + AndroidTargetArch defaultArch; + + /// + /// This field is to be used by the `Resolve` overloads which don't have a way of indicating the desired ABI target for the assembly, but only when the + /// `AndroidTargetArch.None` entry for the assembly in question is **absent**. The field is always set to some value: either the very first assembly added + /// or the one with the `AndroidTargetArch.None` ABI. The latter always wins. + /// + public AssemblyDefinition Default { get; private set; } + public Dictionary Assemblies => assemblies; + + public CacheEntry (TaskLoggingHelper log, string filePath, AssemblyDefinition asm, AndroidTargetArch arch) + { + if (asm == null) { + throw new ArgumentNullException (nameof (asm)); + } + + this.log = log; + Default = asm; + defaultArch = arch; + assemblies = new Dictionary { + { arch, asm }, + }; + } + + public void Add (AndroidTargetArch arch, AssemblyDefinition asm) + { + if (asm == null) { + throw new ArgumentNullException (nameof (asm)); + } + + if (assemblies.ContainsKey (arch)) { + log.LogWarning ($"Entry for assembly '{asm}', architecture '{arch}' already exists. Replacing the old entry."); + } + + assemblies[arch] = asm; + if (arch == AndroidTargetArch.None && defaultArch != AndroidTargetArch.None) { + Default = asm; + defaultArch = arch; + } + } + + void Dispose (bool disposing) + { + if (disposed || !disposing) { + return; + } + + Default = null; + foreach (var kvp in assemblies) { + kvp.Value?.Dispose (); + } + assemblies.Clear (); + disposed = true; + } + + public void Dispose () + { + Dispose (disposing: true); + GC.SuppressFinalize (this); + } + } + + /// + /// Contains a collection of directories where framework assemblies can be found. This collection **must not** + /// contain any directories which contain ABI-specific assemblies. For those, use + /// + public ICollection FrameworkSearchDirectories { get; } = new List (); + + /// + /// Contains a collection of directories where Xamarin.Android (via linker, for instance) has placed the ABI + /// specific assemblies. Each ABI has its own set of directories to search. + /// + public IDictionary> AbiSearchDirectories { get; } = new Dictionary> (); + + readonly List viewStreams = new List (); + bool disposed; + TaskLoggingHelper log; + bool loadDebugSymbols; + ReaderParameters readerParameters; + readonly Dictionary cache; + + public XAAssemblyResolver (TaskLoggingHelper log, bool loadDebugSymbols, ReaderParameters? loadReaderParameters = null) + { + this.log = log; + this.loadDebugSymbols = loadDebugSymbols; + this.readerParameters = loadReaderParameters ?? new ReaderParameters(); + + cache = new Dictionary (StringComparer.OrdinalIgnoreCase); + } + + public AssemblyDefinition? Resolve (string fullName, ReaderParameters? parameters = null) + { + return Resolve (AssemblyNameReference.Parse (fullName), parameters); + } + + public AssemblyDefinition? Resolve (AssemblyNameReference name) + { + return Resolve (name, null); + } + + public AssemblyDefinition? Resolve (AssemblyNameReference name, ReaderParameters? parameters) + { + return Resolve (AndroidTargetArch.None, name, parameters); + } + + public AssemblyDefinition? Resolve (AndroidTargetArch arch, AssemblyNameReference name, ReaderParameters? parameters = null) + { + string shortName = name.Name; + if (cache.TryGetValue (shortName, out CacheEntry? entry)) { + return SelectAssembly (arch, name.FullName, entry, loading: false); + } + + if (arch == AndroidTargetArch.None) { + return FindAndLoadFromDirectories (arch, FrameworkSearchDirectories, name, parameters); + } + + if (!AbiSearchDirectories.TryGetValue (arch, out ICollection? directories) || directories == null) { + throw CreateLoadException (name); + } + + return FindAndLoadFromDirectories (arch, directories, name, parameters); + } + + AssemblyDefinition? FindAndLoadFromDirectories (AndroidTargetArch arch, ICollection directories, AssemblyNameReference name, ReaderParameters? parameters) + { + string? assemblyFile; + foreach (string dir in directories) { + if ((assemblyFile = SearchDirectory (name.Name, dir)) != null) { + return Load (arch, assemblyFile, parameters); + } + } + + return null; + } + + static FileNotFoundException CreateLoadException (AssemblyNameReference name) + { + return new FileNotFoundException ($"Could not load assembly '{name}'."); + } + + static string? SearchDirectory (string name, string directory) + { + if (Path.IsPathRooted (name) && File.Exists (name)) { + return name; + } + + var file = Path.Combine (directory, $"{name}.dll"); + if (File.Exists (file)) { + return file; + } + + return null; + } + + public virtual AssemblyDefinition? Load (AndroidTargetArch arch, string filePath, ReaderParameters? readerParameters = null) + { + string name = Path.GetFileNameWithoutExtension (filePath); + AssemblyDefinition? assembly; + if (cache.TryGetValue (name, out CacheEntry? entry)) { + assembly = SelectAssembly (arch, name, entry, loading: true); + if (assembly != null) { + return assembly; + } + } + + try { + assembly = ReadAssembly (filePath, readerParameters); + } catch (Exception e) when (e is FileNotFoundException || e is DirectoryNotFoundException) { + // These are ok, we can return null + return null; + } + + if (!cache.TryGetValue (name, out entry)) { + entry = new CacheEntry (log, filePath, assembly, arch); + cache.Add (name, entry); + } else { + entry.Add (arch, assembly); + } + + return assembly; + } + + AssemblyDefinition ReadAssembly (string filePath, ReaderParameters? readerParametersOverride = null) + { + ReaderParameters templateParameters = readerParametersOverride ?? this.readerParameters; + bool haveDebugSymbols = loadDebugSymbols && File.Exists (Path.ChangeExtension (filePath, ".pdb")); + var loadReaderParams = new ReaderParameters () { + ApplyWindowsRuntimeProjections = templateParameters.ApplyWindowsRuntimeProjections, + AssemblyResolver = this, + MetadataImporterProvider = templateParameters.MetadataImporterProvider, + InMemory = templateParameters.InMemory, + MetadataResolver = templateParameters.MetadataResolver, + ReadingMode = templateParameters.ReadingMode, + ReadSymbols = haveDebugSymbols, + ReadWrite = templateParameters.ReadWrite, + ReflectionImporterProvider = templateParameters.ReflectionImporterProvider, + SymbolReaderProvider = templateParameters.SymbolReaderProvider, + SymbolStream = templateParameters.SymbolStream, + }; + try { + return LoadFromMemoryMappedFile (filePath, loadReaderParams); + } catch (Exception ex) { + log.LogWarning ($"Failed to read '{filePath}' with debugging symbols. Retrying to load it without it. Error details are logged below."); + log.LogWarning ($"{ex.ToString ()}"); + loadReaderParams.ReadSymbols = false; + return LoadFromMemoryMappedFile (filePath, loadReaderParams); + } + } + + AssemblyDefinition LoadFromMemoryMappedFile (string file, ReaderParameters options) + { + // We can't use MemoryMappedFile when ReadWrite is true + if (options.ReadWrite) { + return AssemblyDefinition.ReadAssembly (file, options); + } + + bool origReadSymbols = options.ReadSymbols; + MemoryMappedViewStream? viewStream = null; + try { + // We must disable reading of symbols, even if they were present, because Cecil is unable to find the symbols file when + // assembly file name is unknown, and this is precisely the case when reading module from a stream. + // Until this issue is resolved, skipping symbol read saves time because reading exception isn't thrown and we don't + // retry the load. + options.ReadSymbols = false; + + // Create stream because CreateFromFile(string, ...) uses FileShare.None which is too strict + using var fileStream = new FileStream (file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, false); + using var mappedFile = MemoryMappedFile.CreateFromFile (fileStream, null, fileStream.Length, MemoryMappedFileAccess.Read, HandleInheritability.None, true); + viewStream = mappedFile.CreateViewStream (0, 0, MemoryMappedFileAccess.Read); + + AssemblyDefinition result = ModuleDefinition.ReadModule (viewStream, options).Assembly; + viewStreams.Add (viewStream); + + // We transferred the ownership of the viewStream to the collection. + viewStream = null; + + return result; + } finally { + options.ReadSymbols = origReadSymbols; + viewStream?.Dispose (); + } + } + + AssemblyDefinition? SelectAssembly (AndroidTargetArch arch, string assemblyName, CacheEntry? entry, bool loading) + { + if (entry == null) { + // Should "never" happen... + throw new ArgumentNullException (nameof (entry)); + } + + if (arch == AndroidTargetArch.None) { + // Disabled for now, generates too much noise. + // if (entry.Assemblies.Count > 1) { + // log.LogWarning ($"Architecture-agnostic entry requested for architecture-specific assembly '{assemblyName}'"); + // } + return entry.Default; + } + + if (!entry.Assemblies.TryGetValue (arch, out AssemblyDefinition? asm)) { + if (loading) { + return null; + } + + if (!entry.Assemblies.TryGetValue (AndroidTargetArch.None, out asm)) { + throw new InvalidOperationException ($"Internal error: assembly '{assemblyName}' for architecture '{arch}' not found in cache entry and architecture-agnostic entry is missing as well"); + } + + if (asm == null) { + throw new InvalidOperationException ($"Internal error: architecture-agnostic cache entry for assembly '{assemblyName}' is null"); + } + + log.LogWarning ($"Returning architecture-agnostic cache entry for assembly '{assemblyName}'. Requested architecture was: {arch}"); + return asm; + } + + if (asm == null) { + throw new InvalidOperationException ($"Internal error: null reference for assembly '{assemblyName}' in assembly cache entry"); + } + + return asm; + } + + public void Dispose () + { + Dispose (disposing: true); + GC.SuppressFinalize (this); + } + + protected virtual void Dispose (bool disposing) + { + if (disposed || !disposing) { + return; + } + + foreach (var kvp in cache) { + kvp.Value?.Dispose (); + } + cache.Clear (); + + foreach (MemoryMappedViewStream viewStream in viewStreams) { + viewStream.Dispose (); + } + viewStreams.Clear (); + + disposed = true; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs new file mode 100644 index 00000000000..5f965ab46ff --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs @@ -0,0 +1,120 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; + +using Java.Interop.Tools.Cecil; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Mono.Cecil; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +class JavaType +{ + public readonly TypeDefinition Type; + public readonly IDictionary? PerAbiTypes; + public bool IsABiSpecific { get; } + + public JavaType (TypeDefinition type, IDictionary? perAbiTypes) + { + Type = type; + if (perAbiTypes != null) { + PerAbiTypes = new ReadOnlyDictionary (perAbiTypes); + IsABiSpecific = perAbiTypes.Count > 1 || (perAbiTypes.Count == 1 && !perAbiTypes.ContainsKey (AndroidTargetArch.None)); + } + } +} + +class XAJavaTypeScanner +{ + sealed class TypeData + { + public readonly TypeDefinition FirstType; + public readonly Dictionary PerAbi; + + public bool IsAbiSpecific => !PerAbi.ContainsKey (AndroidTargetArch.None); + + public TypeData (TypeDefinition firstType) + { + FirstType = firstType; + PerAbi = new Dictionary (); + } + } + + public bool ErrorOnCustomJavaObject { get; set; } + + TaskLoggingHelper log; + TypeDefinitionCache cache; + + public XAJavaTypeScanner (TaskLoggingHelper log, TypeDefinitionCache cache) + { + this.log = log; + this.cache = cache; + } + + public List GetJavaTypes (ICollection inputAssemblies, XAAssemblyResolver resolver) + { + var types = new Dictionary (StringComparer.Ordinal); + foreach (ITaskItem asmItem in inputAssemblies) { + AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (asmItem); + AssemblyDefinition asmdef = resolver.Load (arch, asmItem.ItemSpec); + + foreach (ModuleDefinition md in asmdef.Modules) { + foreach (TypeDefinition td in md.Types) { + AddJavaType (td, types, arch); + } + } + } + + var ret = new List (); + foreach (var kvp in types) { + ret.Add (new JavaType (kvp.Value.FirstType, kvp.Value.IsAbiSpecific ? kvp.Value.PerAbi : null)); + } + + return ret; + } + + void AddJavaType (TypeDefinition type, Dictionary types, AndroidTargetArch arch) + { + if (type.IsSubclassOf ("Java.Lang.Object", cache) || type.IsSubclassOf ("Java.Lang.Throwable", cache) || (type.IsInterface && type.ImplementsInterface ("Java.Interop.IJavaPeerable", cache))) { + // For subclasses of e.g. Android.App.Activity. + string typeName = type.GetPartialAssemblyQualifiedName (cache); + if (!types.TryGetValue (typeName, out TypeData typeData)) { + typeData = new TypeData (type); + types.Add (typeName, typeData); + } + + if (typeData.PerAbi.ContainsKey (AndroidTargetArch.None)) { + if (arch == AndroidTargetArch.None) { + throw new InvalidOperationException ($"Duplicate type '{type.FullName}' in assembly {type.Module.FileName}"); + } + + throw new InvalidOperationException ($"Previously added type '{type.FullName}' was in ABI-agnostic assembly, new one comes from ABI {arch} assembly"); + } + + if (typeData.PerAbi.ContainsKey (arch)) { + throw new InvalidOperationException ($"Duplicate type '{type.FullName}' in assembly {type.Module.FileName}, for ABI {arch}"); + } + + typeData.PerAbi.Add (arch, type); + } else if (type.IsClass && !type.IsSubclassOf ("System.Exception", cache) && type.ImplementsInterface ("Android.Runtime.IJavaObject", cache)) { + string message = $"XA4212: Type `{type.FullName}` implements `Android.Runtime.IJavaObject` but does not inherit `Java.Lang.Object` or `Java.Lang.Throwable`. This is not supported."; + + if (ErrorOnCustomJavaObject) { + log.LogError (message); + } else { + log.LogWarning (message); + } + return; + } + + if (!type.HasNestedTypes) { + return; + } + + foreach (TypeDefinition nested in type.NestedTypes) { + AddJavaType (nested, types, arch); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index b118454ee96..3bf64758b7c 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -335,7 +335,7 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. false true True - True + False False <_AndroidUseMarshalMethods Condition=" '$(AndroidIncludeDebugSymbols)' == 'True' ">False <_AndroidUseMarshalMethods Condition=" '$(AndroidIncludeDebugSymbols)' != 'True' ">$(AndroidEnableMarshalMethods) diff --git a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs index 72133c28f25..e6331a607d1 100644 --- a/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/InstallAndRunTests.cs @@ -29,6 +29,11 @@ public void Teardown () [Test] public void NativeAssemblyCacheWithSatelliteAssemblies ([Values (true, false)] bool enableMarshalMethods) { + // TODO: enable when marshal methods are fixed + if (enableMarshalMethods) { + Assert.Ignore ("Test is skipped when marshal methods are enabled, pending fixes to MM for .NET9"); + } + var path = Path.Combine ("temp", TestName); var lib = new XamarinAndroidLibraryProject { ProjectName = "Localization",