From 5cdfa147e24ea54836a22586fe94409ff940ef8a Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 14 Jan 2020 22:05:17 +0100 Subject: [PATCH] New code to perform managed <-> java lookups (typemap) Xamarin.Assembly needs to "translate" managed types to Java types and vice versa in order to provide a bridge between the two world. So far it has been done using a straightforward (and fast) method of performing the lookups - all the type pairs were stored in two tables of the same size, with all type names padded to the width of the longest name so that the `bsearch` C function can be used to quickly perform a binary search over the data set. This approach works very well at the expense of data size (shorter strings are 0-padded to the maximum width) and a slightly degraded performace because of the requirement to perform string comparisons. Furthermore, the lookup required that reflection is used to obtain full managed type name (when translating from managed to Java) or to get a `Type` instance from type name (when translating from Java to managed). For Release builds all the above data is placed in the `libxamarin-app.so` library, for Debug builds it is also placed in two files - one for each direction of lookup, described above. This commit is a slight improvement over the above scheme. It eliminates reflection from the process by using managed type tokens (which are integers) and using UUID/Guid of the module in which the type is found. This allows us to perform the binary search over the set of 20 bytes (16 bytes for the UUID and 4 bytes for the token ID) for managed to Java lookups and a single string comparison + binary search over a set of integers for the Java to managed lookup. Java type names must still be used because Java doesn't provide any equivalent to the .NET's type token and module UUID. Those names are still 0-padded to the width of the longest name but there are no longer duplicated. Managed type names are eliminated completely. If Xamarin.Android Instant Run is not used (which is the case for OSS code) for Debug builds, the operation is performed in the same way for both Release and Debug builds. If, however, Instant Run is in effect, the type maps are stored in several files with the .typemap extension - one per **module**. The files contain both the Java to managed maps as well as managed to Java maps (which use indexes into the Java to managed maps). All of those files are loaded during Debug app startup and used to construct a dataset which is the searched during all the lookups. Performance changes: TBD Size changes: TBD --- .external | 2 +- .../Android.Runtime/AndroidRuntime.cs | 2 +- src/Mono.Android/Android.Runtime/JNIEnv.cs | 4 +- src/Mono.Android/Java.Interop/TypeManager.cs | 13 +- .../Tasks/GenerateJavaStubs.cs | 73 --- .../Tasks/GeneratePackageManagerJava.cs | 13 +- .../Tasks/GenerateTypeMaps.cs | 89 +++ .../Utilities/EnvironmentHelper.cs | 27 +- .../ARMNativeAssemblerTargetProvider.cs | 6 + ...pplicationConfigNativeAssemblyGenerator.cs | 26 +- .../NativeAssemblerTargetProvider.cs | 20 +- .../Utilities/NativeAssemblyDataStream.cs | 235 -------- .../Utilities/NativeAssemblyGenerator.cs | 132 +++- .../Utilities/NativeTypeMappingData.cs | 44 ++ .../Utilities/TypeMapGenerator.cs | 386 ++++++++++++ .../TypeMappingNativeAssemblyGenerator.cs | 257 ++++++-- .../X86NativeAssemblerTargetProvider.cs | 6 + .../Xamarin.Android.Build.Tasks.csproj | 11 +- .../Xamarin.Android.Common.targets | 48 +- src/monodroid/jni/android-system.cc | 23 +- src/monodroid/jni/android-system.hh | 3 +- src/monodroid/jni/application_dso_stub.cc | 14 +- src/monodroid/jni/embedded-assemblies-zip.cc | 13 - src/monodroid/jni/embedded-assemblies.cc | 567 ++++++++++++++---- src/monodroid/jni/embedded-assemblies.hh | 28 +- src/monodroid/jni/external-api.cc | 10 +- src/monodroid/jni/monodroid-glue-internal.hh | 1 + src/monodroid/jni/monodroid-glue.cc | 24 +- src/monodroid/jni/xamarin-app.h | 42 -- src/monodroid/jni/xamarin-app.hh | 88 +++ 30 files changed, 1563 insertions(+), 644 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Tasks/GenerateTypeMaps.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblyDataStream.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/NativeTypeMappingData.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs delete mode 100644 src/monodroid/jni/xamarin-app.h create mode 100644 src/monodroid/jni/xamarin-app.hh diff --git a/.external b/.external index 102c0a46263..f924d11bba8 100644 --- a/.external +++ b/.external @@ -1,2 +1,2 @@ -xamarin/monodroid:master@0f6725edfa09559afad58233d43f385122e31e8d +grendello/monodroid:new-typemap@efe3956ce2e557b5f8286f984731e43c6ee402f7 mono/mono:2019-10@18920a83f423fb864a2263948737681968f5b2c8 diff --git a/src/Mono.Android/Android.Runtime/AndroidRuntime.cs b/src/Mono.Android/Android.Runtime/AndroidRuntime.cs index 93a5fc0e9de..da6af9f62a8 100644 --- a/src/Mono.Android/Android.Runtime/AndroidRuntime.cs +++ b/src/Mono.Android/Android.Runtime/AndroidRuntime.cs @@ -234,7 +234,7 @@ protected override IEnumerable GetSimpleReferences (Type type) foreach (var simpleRef in base.GetSimpleReferences (type)) { yield return simpleRef; } - var j = JNIEnv.monodroid_typemap_managed_to_java (type.FullName + ", " + type.Assembly.GetName ().Name); + var j = JNIEnv.monodroid_typemap_managed_to_java (type.Module.ModuleVersionId.ToByteArray (), type.MetadataToken); if (j != IntPtr.Zero) { yield return Marshal.PtrToStringAnsi (j); } diff --git a/src/Mono.Android/Android.Runtime/JNIEnv.cs b/src/Mono.Android/Android.Runtime/JNIEnv.cs index 69cbc533d32..c08ce020441 100644 --- a/src/Mono.Android/Android.Runtime/JNIEnv.cs +++ b/src/Mono.Android/Android.Runtime/JNIEnv.cs @@ -632,13 +632,13 @@ public static string GetClassNameFromInstance (IntPtr jobject) } [DllImport ("__Internal", CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr monodroid_typemap_managed_to_java (string managed); + internal static extern IntPtr monodroid_typemap_managed_to_java (byte[] mvid, int token); public static string GetJniName (Type type) { if (type == null) throw new ArgumentNullException ("type"); - var java = monodroid_typemap_managed_to_java (type.FullName + ", " + type.Assembly.GetName ().Name); + var java = monodroid_typemap_managed_to_java (type.Module.ModuleVersionId.ToByteArray (), type.MetadataToken); return java == IntPtr.Zero ? JavaNativeTypeManager.ToJniName (type) : Marshal.PtrToStringAnsi (java); diff --git a/src/Mono.Android/Java.Interop/TypeManager.cs b/src/Mono.Android/Java.Interop/TypeManager.cs index 0cf4265f62c..bc6af172a0c 100644 --- a/src/Mono.Android/Java.Interop/TypeManager.cs +++ b/src/Mono.Android/Java.Interop/TypeManager.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Reflection; using System.Linq; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using Java.Interop.Tools.TypeNameMappings; @@ -203,14 +204,14 @@ static Exception CreateJavaLocationException () return new JavaLocationException (loc.ToString ()); } - [DllImport ("__Internal", CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr monodroid_typemap_java_to_managed (string java); + [MethodImplAttribute(MethodImplOptions.InternalCall)] + static extern Type monodroid_typemap_java_to_managed (string java_type_name); internal static Type GetJavaToManagedType (string class_name) { - var t = monodroid_typemap_java_to_managed (class_name); - if (t != IntPtr.Zero) - return Type.GetType (Marshal.PtrToStringAnsi (t)); + Type type = monodroid_typemap_java_to_managed (class_name); + if (type != null) + return type; if (!JNIEnv.IsRunningOnDesktop) { return null; @@ -218,7 +219,7 @@ internal static Type GetJavaToManagedType (string class_name) __TypeRegistrations.RegisterPackages (); - var type = (Type) null; + type = null; int ls = class_name.LastIndexOf ('/'); var package = ls >= 0 ? class_name.Substring (0, ls) : ""; List> mappers; diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 52270a8c7a5..4d94c095943 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -147,8 +147,6 @@ void Run (DirectoryAssemblyResolver res) }; var all_java_types = scanner.GetJavaTypes (assemblies, res); - WriteTypeMappings (all_java_types); - var java_types = all_java_types .Where (t => !JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (t)) .ToArray (); @@ -300,76 +298,5 @@ void SaveResource (string resource, string filename, string destDir, Func types) - { - void logger (TraceLevel level, string value) => Log.LogDebugMessage (value); - TypeNameMapGenerator createTypeMapGenerator () => UseSharedRuntime ? - new TypeNameMapGenerator (types, logger) : - new TypeNameMapGenerator (ResolvedAssemblies.Select (p => p.ItemSpec), logger); - using (var gen = createTypeMapGenerator ()) { - using (var ms = new MemoryStream ()) { - UpdateWhenChanged (Path.Combine (OutputDirectory, "typemap.jm"), "jm", ms, gen.WriteJavaToManaged); - UpdateWhenChanged (Path.Combine (OutputDirectory, "typemap.mj"), "mj", ms, gen.WriteManagedToJava); - } - } - } - - void UpdateWhenChanged (string path, string type, MemoryStream ms, Action generator) - { - if (!EmbedAssemblies) { - ms.SetLength (0); - generator (ms); - MonoAndroidHelper.CopyIfStreamChanged (ms, path); - } - - string dataFilePath = $"{path}.inc"; - using (var stream = new NativeAssemblyDataStream ()) { - if (EmbedAssemblies) { - generator (stream); - stream.EndOfFile (); - MonoAndroidHelper.CopyIfStreamChanged (stream, dataFilePath); - } else { - stream.EmptyFile (); - } - - var generatedFiles = new List (); - string mappingFieldName = $"{type}_typemap"; - string dataFileName = Path.GetFileName (dataFilePath); - NativeAssemblerTargetProvider asmTargetProvider; - var utf8Encoding = new UTF8Encoding (false); - foreach (string abi in SupportedAbis) { - ms.SetLength (0); - switch (abi.Trim ()) { - case "armeabi-v7a": - asmTargetProvider = new ARMNativeAssemblerTargetProvider (is64Bit: false); - break; - - case "arm64-v8a": - asmTargetProvider = new ARMNativeAssemblerTargetProvider (is64Bit: true); - break; - - case "x86": - asmTargetProvider = new X86NativeAssemblerTargetProvider (is64Bit: false); - break; - - case "x86_64": - asmTargetProvider = new X86NativeAssemblerTargetProvider (is64Bit: true); - break; - - default: - throw new InvalidOperationException ($"Unknown ABI {abi}"); - } - - var asmgen = new TypeMappingNativeAssemblyGenerator (asmTargetProvider, stream, dataFileName, stream.MapByteCount, mappingFieldName); - asmgen.EmbedAssemblies = EmbedAssemblies; - string asmFileName = $"{path}.{abi.Trim ()}.s"; - using (var sw = new StreamWriter (ms, utf8Encoding, bufferSize: 8192, leaveOpen: true)) { - asmgen.Write (sw, dataFileName); - MonoAndroidHelper.CopyIfStreamChanged (ms, asmFileName); - } - } - } - } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 45a59759110..2764f083cc2 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -58,6 +58,9 @@ public class GeneratePackageManagerJava : AndroidTask [Required] public bool EnablePreloadAssembliesDefault { get; set; } + [Required] + public bool InstantRunEnabled { get; set; } + public string BoundExceptionType { get; set; } public string PackageNamingPolicy { get; set; } @@ -252,7 +255,8 @@ void AddEnvironment () foreach (string abi in SupportedAbis) { ms.SetLength (0); NativeAssemblerTargetProvider asmTargetProvider; - string asmFileName = Path.Combine (EnvironmentOutputDirectory, $"environment.{abi.ToLowerInvariant ()}.s"); + string baseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"environment.{abi.ToLowerInvariant ()}"); + string asmFilePath = $"{baseAsmFilePath}.s"; switch (abi.Trim ()) { case "armeabi-v7a": asmTargetProvider = new ARMNativeAssemblerTargetProvider (false); @@ -274,7 +278,7 @@ void AddEnvironment () throw new InvalidOperationException ($"Unknown ABI {abi}"); } - var asmgen = new ApplicationConfigNativeAssemblyGenerator (asmTargetProvider, environmentVariables, systemProperties) { + var asmgen = new ApplicationConfigNativeAssemblyGenerator (asmTargetProvider, baseAsmFilePath, environmentVariables, systemProperties) { IsBundledApp = IsBundledApplication, UsesMonoAOT = usesMonoAOT, UsesMonoLLVM = EnableLLVM, @@ -284,11 +288,12 @@ void AddEnvironment () BrokenExceptionTransitions = brokenExceptionTransitions, PackageNamingPolicy = pnp, BoundExceptionType = boundExceptionType, + InstantRunEnabled = InstantRunEnabled, }; using (var sw = new StreamWriter (ms, utf8Encoding, bufferSize: 8192, leaveOpen: true)) { - asmgen.Write (sw, asmFileName); - MonoAndroidHelper.CopyIfStreamChanged (ms, asmFileName); + asmgen.Write (sw); + MonoAndroidHelper.CopyIfStreamChanged (ms, asmFilePath); } } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateTypeMaps.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateTypeMaps.cs new file mode 100644 index 00000000000..7f34905cc3e --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateTypeMaps.cs @@ -0,0 +1,89 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; + +using Microsoft.Build.Framework; + +using Java.Interop.Tools.Cecil; +using Java.Interop.Tools.Diagnostics; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks +{ + public class GenerateTypeMaps : AndroidTask + { + public override string TaskPrefix => "GTM"; + + [Required] + public ITaskItem[] ResolvedAssemblies { get; set; } + + [Required] + public ITaskItem[] ResolvedUserAssemblies { get; set; } + + [Required] + public ITaskItem [] FrameworkDirectories { get; set; } + + [Required] + public string[] SupportedAbis { get; set; } + + [Required] + public string OutputDirectory { get; set; } + + [Required] + public bool GenerateNativeAssembly { get; set; } + + [Output] + public string[] GeneratedBinaryTypeMaps { get; set; } + + public bool ErrorOnCustomJavaObject { get; set; } + + public override bool RunTask () + { + try { + Run (); + } catch (XamarinAndroidException e) { + Log.LogCodedError (string.Format ("XA{0:0000}", e.Code), e.MessageWithoutCode); + if (MonoAndroidHelper.LogInternalExceptions) + Log.LogMessage (e.ToString ()); + } + + if (Log.HasLoggedErrors) { + // Ensure that on a rebuild, we don't *skip* the `_GenerateJavaStubs` target, + // by ensuring that the target outputs have been deleted. + Files.DeleteFile (Path.Combine (OutputDirectory, "typemap.index"), Log); + foreach (string file in Directory.EnumerateFiles (OutputDirectory, "*.typemap")) { + Files.DeleteFile (file, Log); + } + } + + return !Log.HasLoggedErrors; + } + + void Run () + { + var interestingAssemblies = new HashSet (StringComparer.OrdinalIgnoreCase); + + var res = new DirectoryAssemblyResolver (this.CreateTaskLogger (), loadDebugSymbols: true); + foreach (var dir in FrameworkDirectories) { + if (Directory.Exists (dir.ItemSpec)) + res.SearchDirectories.Add (dir.ItemSpec); + } + + foreach (ITaskItem assembly in ResolvedAssemblies) { + res.Load (assembly.ItemSpec); + if (String.Compare ("MonoAndroid", assembly.GetMetadata ("TargetFrameworkIdentifier"), StringComparison.Ordinal) != 0) + continue; + if (interestingAssemblies.Contains (assembly.ItemSpec)) + continue; + interestingAssemblies.Add (assembly.ItemSpec); + } + + var tmg = new TypeMapGenerator ((string message) => Log.LogDebugMessage (message), SupportedAbis); + if (!tmg.Generate (res, interestingAssemblies, OutputDirectory, GenerateNativeAssembly)) + throw new XamarinAndroidException (99999, "Failed to generate type maps"); + GeneratedBinaryTypeMaps = tmg.GeneratedBinaryTypeMaps.ToArray (); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs index d253dc90264..38f796f6b2f 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/EnvironmentHelper.cs @@ -22,13 +22,14 @@ public sealed class ApplicationConfig public bool uses_assembly_preload; public bool is_a_bundled_app; public bool broken_exception_transitions; + public bool instant_run_enabled; public byte bound_stream_io_exception_type; public uint package_naming_policy; public uint environment_variable_count; public uint system_property_count; public string android_package_name; }; - const uint ApplicationConfigFieldCount = 10; + const uint ApplicationConfigFieldCount = 11; static readonly object ndkInitLock = new object (); static readonly char[] readElfFieldSeparator = new [] { ' ', '\t' }; @@ -49,10 +50,11 @@ public sealed class ApplicationConfig "app_environment_variables", "app_system_properties", "application_config", - "jm_typemap", - "jm_typemap_header", - "mj_typemap", - "mj_typemap_header", + "map_modules", + "map_module_count", + "java_type_count", + "java_name_width", + "map_java", "mono_aot_mode_name", }; @@ -136,27 +138,32 @@ static ApplicationConfig ReadApplicationConfig (string envFile) ret.broken_exception_transitions = ConvertFieldToBool ("broken_exception_transitions", envFile, i, field [1]); break; - case 5: // bound_stream_io_exception_type: byte / .byte + case 5: // instant_run_enabled: bool / .byte + AssertFieldType (envFile, ".byte", field [0], i); + ret.instant_run_enabled = ConvertFieldToBool ("instant_run_enabled", envFile, i, field [1]); + break; + + case 6: // bound_stream_io_exception_type: byte / .byte AssertFieldType (envFile, ".byte", field [0], i); ret.bound_stream_io_exception_type = ConvertFieldToByte ("bound_stream_io_exception_type", envFile, i, field [1]); break; - case 6: // package_naming_policy: uint32_t / .word | .long + case 7: // package_naming_policy: uint32_t / .word | .long Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}"); ret.package_naming_policy = ConvertFieldToUInt32 ("package_naming_policy", envFile, i, field [1]); break; - case 7: // environment_variable_count: uint32_t / .word | .long + case 8: // environment_variable_count: uint32_t / .word | .long Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}"); ret.environment_variable_count = ConvertFieldToUInt32 ("environment_variable_count", envFile, i, field [1]); break; - case 8: // system_property_count: uint32_t / .word | .long + case 9: // system_property_count: uint32_t / .word | .long Assert.IsTrue (expectedUInt32Types.Contains (field [0]), $"Unexpected uint32_t field type in '{envFile}:{i}': {field [0]}"); ret.system_property_count = ConvertFieldToUInt32 ("system_property_count", envFile, i, field [1]); break; - case 9: // android_package_name: string / [pointer type] + case 10: // android_package_name: string / [pointer type] Assert.IsTrue (expectedPointerTypes.Contains (field [0]), $"Unexpected pointer field type in '{envFile}:{i}': {field [0]}"); pointers.Add (field [1].Trim ()); break; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ARMNativeAssemblerTargetProvider.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ARMNativeAssemblerTargetProvider.cs index 2791da006a0..e563f64ffe6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ARMNativeAssemblerTargetProvider.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ARMNativeAssemblerTargetProvider.cs @@ -5,9 +5,15 @@ namespace Xamarin.Android.Tasks { class ARMNativeAssemblerTargetProvider : NativeAssemblerTargetProvider { + const string ARMV7a = "armeabi-v7a"; + const string ARMV8a = "arm64-v8a"; + public override bool Is64Bit { get; } public override string PointerFieldType { get; } public override string TypePrefix { get; } + public override string AbiName => Is64Bit ? ARMV8a : ARMV7a; + public override uint MapModulesAlignBits => Is64Bit ? 3u : 2u; + public override uint MapJavaAlignBits { get; } = 2; public ARMNativeAssemblerTargetProvider (bool is64Bit) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs index 456db8704ba..16bca1d0a1e 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs @@ -8,6 +8,8 @@ namespace Xamarin.Android.Tasks { class ApplicationConfigNativeAssemblyGenerator : NativeAssemblyGenerator { + readonly uint fieldAlignBytes; + SortedDictionary environmentVariables; SortedDictionary systemProperties; uint stringCounter = 0; @@ -20,16 +22,19 @@ class ApplicationConfigNativeAssemblyGenerator : NativeAssemblyGenerator public string AndroidPackageName { get; set; } public bool BrokenExceptionTransitions { get; set; } public global::Android.Runtime.BoundExceptionType BoundExceptionType { get; set; } + public bool InstantRunEnabled { get; set; } public PackageNamingPolicy PackageNamingPolicy { get; set; } - public ApplicationConfigNativeAssemblyGenerator (NativeAssemblerTargetProvider targetProvider, IDictionary environmentVariables, IDictionary systemProperties) - : base (targetProvider) + public ApplicationConfigNativeAssemblyGenerator (NativeAssemblerTargetProvider targetProvider, string baseFileName, IDictionary environmentVariables, IDictionary systemProperties) + : base (targetProvider, baseFileName) { if (environmentVariables != null) this.environmentVariables = new SortedDictionary (environmentVariables, StringComparer.Ordinal); if (systemProperties != null) this.systemProperties = new SortedDictionary (systemProperties, StringComparer.Ordinal); + + fieldAlignBytes = targetProvider.Is64Bit ? 8u : 4u; } protected override void WriteSymbols (StreamWriter output) @@ -44,7 +49,7 @@ protected override void WriteSymbols (StreamWriter output) WriteData (output, AndroidPackageName, stringLabel); WriteDataSection (output, "application_config"); - WriteSymbol (output, "application_config", TargetProvider.GetStructureAlignment (true), fieldAlignBytes: 4, isGlobal: true, alwaysWriteSize: true, structureWriter: () => { + WriteSymbol (output, "application_config", TargetProvider.GetStructureAlignment (true), fieldAlignBytes, isGlobal: true, alwaysWriteSize: true, structureWriter: () => { // Order of fields and their type must correspond *exactly* to that in // src/monodroid/jni/xamarin-app.h ApplicationConfig structure WriteCommentLine (output, "uses_mono_llvm"); @@ -62,6 +67,9 @@ protected override void WriteSymbols (StreamWriter output) WriteCommentLine (output, "broken_exception_transitions"); size += WriteData (output, BrokenExceptionTransitions); + WriteCommentLine (output, "instant_run_enabled"); + size += WriteData (output, InstantRunEnabled); + WriteCommentLine (output, "bound_exception_type"); size += WriteData (output, (byte)BoundExceptionType); @@ -80,7 +88,7 @@ protected override void WriteSymbols (StreamWriter output) } WriteCommentLine (output, "android_package_name"); - size += WritePointer (output, stringLabel); + size += WritePointer (output, MakeLocalLabel (stringLabel)); return size; }); @@ -88,7 +96,7 @@ protected override void WriteSymbols (StreamWriter output) stringLabel = GetStringLabel (); WriteData (output, MonoAOTMode ?? String.Empty, stringLabel); WriteDataSection (output, "mono_aot_mode_name"); - WritePointer (output, stringLabel, "mono_aot_mode_name", isGlobal: true); + WritePointer (output, MakeLocalLabel (stringLabel), "mono_aot_mode_name", isGlobal: true); WriteNameValueStringArray (output, "app_environment_variables", environmentVariables); WriteNameValueStringArray (output, "app_system_properties", systemProperties); @@ -98,7 +106,7 @@ void WriteNameValueStringArray (StreamWriter output, string label, SortedDiction { if (entries == null || entries.Count == 0) { WriteDataSection (output, label); - WriteSymbol (output, label, TargetProvider.GetStructureAlignment (true), fieldAlignBytes: 4, isGlobal: true, alwaysWriteSize: true, structureWriter: null); + WriteSymbol (output, label, TargetProvider.GetStructureAlignment (true), fieldAlignBytes, isGlobal: true, alwaysWriteSize: true, structureWriter: null); return; } @@ -117,11 +125,11 @@ void WriteNameValueStringArray (StreamWriter output, string label, SortedDiction } WriteDataSection (output, label); - WriteSymbol (output, label, TargetProvider.GetStructureAlignment (true), fieldAlignBytes: 4, isGlobal: true, alwaysWriteSize: true, structureWriter: () => { + WriteSymbol (output, label, TargetProvider.GetStructureAlignment (true), fieldAlignBytes, isGlobal: true, alwaysWriteSize: true, structureWriter: () => { uint size = 0; foreach (string l in entry_labels) { - size += WritePointer (output, l); + size += WritePointer (output, MakeLocalLabel (l)); } return size; @@ -136,7 +144,7 @@ void WriteDataSection (StreamWriter output, string tag) string GetStringLabel () { stringCounter++; - return $".L.str.{stringCounter}"; + return $"env.str.{stringCounter}"; } }; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblerTargetProvider.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblerTargetProvider.cs index d3158abf8fc..6b379232078 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblerTargetProvider.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblerTargetProvider.cs @@ -8,6 +8,9 @@ abstract class NativeAssemblerTargetProvider public abstract bool Is64Bit { get; } public abstract string PointerFieldType { get; } public abstract string TypePrefix { get; } + public abstract string AbiName { get; } + public abstract uint MapModulesAlignBits { get; } + public abstract uint MapJavaAlignBits { get; } public virtual string MapType () { @@ -30,22 +33,27 @@ abstract class NativeAssemblerTargetProvider public virtual uint GetTypeSize () { - if (typeof(T) == typeof(byte)) + return GetTypeSize (typeof(T)); + } + + public virtual uint GetTypeSize (Type type) + { + if (type == typeof(byte)) return 1u; - if (typeof(T) == typeof(bool)) + if (type == typeof(bool)) return 1u; - if (typeof(T) == typeof(string)) + if (type == typeof(string)) return GetPointerSize(); - if (typeof(T) == typeof(Int32) || typeof(T) == typeof(UInt32)) + if (type == typeof(Int32) || type == typeof(UInt32)) return 4u; - if (typeof(T) == typeof(Int64) || typeof(T) == typeof(UInt64)) + if (type == typeof(Int64) || type == typeof(UInt64)) return 8u; - throw new InvalidOperationException ($"Unable to map managed type {typeof(T)} to native assembly type"); + throw new InvalidOperationException ($"Unable to map managed type {type} to native assembly type"); } public virtual uint GetStructureAlignment (bool hasPointers) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblyDataStream.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblyDataStream.cs deleted file mode 100644 index f0e3e6526a4..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblyDataStream.cs +++ /dev/null @@ -1,235 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Security.Cryptography; -using System.IO; -using System.Text; -using Java.Interop.Tools.JavaCallableWrappers; - -namespace Xamarin.Android.Tasks -{ - class NativeAssemblyDataStream : Stream - { - const uint MapVersionFound = 0x0001; - const uint MapEntryCountFound = 0x0002; - const uint MapEntryLengthFound = 0x0004; - const uint MapValueOffsetFound = 0x0008; - const uint MapEverythingFound = MapVersionFound | MapEntryCountFound | MapEntryLengthFound | MapValueOffsetFound; - - const string MapFieldVersion = "version"; - const string MapFieldEntryCount = "entry-count"; - const string MapFieldEntryLength = "entry-len"; - const string MapFieldValueOffset = "value-offset"; - - const uint MaxFieldsInRow = 32; - - MemoryStream outputStream; - StreamWriter outputWriter; - bool firstWrite = true; - uint rowFieldCounter; - HashAlgorithm hashAlgorithm; - bool hashingFinished; - uint byteCount = 0; - - public int MapVersion { get; set; } = -1; - public int MapEntryCount { get; set; } = -1; - public int MapEntryLength { get; set; } = -1; - public int MapValueOffset { get; set; } = -1; - public uint MapByteCount => byteCount; - - public override bool CanRead => outputStream.CanRead; - public override bool CanSeek => outputStream.CanSeek; - public override bool CanWrite => outputStream.CanWrite; - public override long Length => outputStream.Length; - - public override long Position { - get => outputStream.Position; - set => outputStream.Position = value; - } - - public NativeAssemblyDataStream () - { - outputStream = new MemoryStream (); - outputWriter = new StreamWriter (outputStream, new UTF8Encoding (false)); - hashAlgorithm = new Crc64 (); - hashAlgorithm.Initialize (); - } - - public byte[] GetStreamHash () - { - if (!hashingFinished) { - hashAlgorithm.TransformFinalBlock (new byte[0], 0, 0); - hashingFinished = true; - } - - return hashAlgorithm.Hash; - } - - protected override void Dispose (bool disposing) - { - base.Dispose (disposing); - if (!disposing) - return; - - outputWriter.Dispose (); - outputWriter = null; - outputStream = null; - } - - public void EndOfFile () - { - outputWriter.WriteLine (); - Flush (); - } - - public void EmptyFile () - { - MapVersion = 1; - MapEntryCount = 0; - MapEntryLength = 0; - MapValueOffset = 0; - } - - public override void Flush () - { - outputWriter.Flush (); - outputStream.Flush (); - } - - public override int Read (byte[] buffer, int offset, int count) - { - return outputStream.Read (buffer, offset, count); - } - - public override long Seek (long offset, SeekOrigin origin) - { - return outputStream.Seek (offset, origin); - } - - public override void SetLength (long value) - { - outputStream.SetLength (value); - } - - public override void Write (byte[] buffer, int offset, int count) - { - if (buffer == null) - throw new ArgumentNullException (nameof (buffer)); - if (offset < 0) - throw new ArgumentOutOfRangeException (nameof (offset)); - if (count < 0) - throw new ArgumentOutOfRangeException (nameof (count)); - if (offset + count > buffer.Length) - throw new ArgumentException ($"The sum of the '{nameof (offset)}' and '{nameof (count)}' arguments is greater than the buffer length"); - if (outputWriter == null) - throw new ObjectDisposedException (this.GetType ().Name); - - if (!hashingFinished) - hashAlgorithm.TransformBlock (buffer, offset, count, null, 0); - - if (firstWrite) { - // Kind of a backward thing to do, but I wanted to avoid having to modfy (and bump the - // submodule of) Java.Interop. This should be Safe Enough™ (until it's not) - string line = Encoding.UTF8.GetString (buffer, offset, count); - - // Format used by Java.Interop.Tools.JavaCallableWrappers.TypeNameMapGenerator: - // - // "version=1\u0000entry-count={0}\u0000entry-len={1}\u0000value-offset={2}\u0000" - // - string[] parts = line.Split ('\u0000'); - if (parts.Length != 5) - HeaderFormatError ("invalid number of fields"); - - // Let's be just slightly flexible :P - uint foundMask = 0; - foreach (string p in parts) { - string field = p.Trim (); - if (String.IsNullOrEmpty (field)) - continue; - - string[] fieldParts = field.Split(new char[]{'='}, 2); - if (fieldParts.Length != 2) - HeaderFormatError ($"invalid field format '{field}'"); - - switch (fieldParts [0]) { - case MapFieldVersion: - foundMask |= MapVersionFound; - MapVersion = GetHeaderInteger (fieldParts[0], fieldParts [1]); - break; - - case MapFieldEntryCount: - foundMask |= MapEntryCountFound; - MapEntryCount = GetHeaderInteger (fieldParts[0], fieldParts [1]); - break; - - case MapFieldEntryLength: - foundMask |= MapEntryLengthFound; - MapEntryLength = GetHeaderInteger (fieldParts[0], fieldParts [1]); - break; - - case MapFieldValueOffset: - foundMask |= MapValueOffsetFound; - MapValueOffset = GetHeaderInteger (fieldParts[0], fieldParts [1]); - break; - - default: - HeaderFormatError ($"unknown field '{fieldParts [0]}'"); - break; - } - } - - if (foundMask != MapEverythingFound) { - var missingFields = new List (); - if ((foundMask & MapVersionFound) == 0) - missingFields.Add (MapFieldVersion); - if ((foundMask & MapEntryCountFound) == 0) - missingFields.Add (MapFieldEntryCount); - if ((foundMask & MapEntryLengthFound) == 0) - missingFields.Add (MapFieldEntryLength); - if ((foundMask & MapValueOffsetFound) == 0) - missingFields.Add (MapFieldValueOffset); - - var sb = new StringBuilder ("missing header field"); - if (missingFields.Count > 1) - sb.Append ('s'); - sb.Append (": "); - sb.Append (String.Join (", ", missingFields)); - HeaderFormatError (sb.ToString ()); - } - - firstWrite = false; - rowFieldCounter = 0; - return; - } - - for (int i = 0; i < count; i++) { - byteCount++; - if (rowFieldCounter == 0) - outputWriter.Write ("\t.byte "); - - byte b = buffer [offset + i]; - if (rowFieldCounter > 0) - outputWriter.Write (", "); - outputWriter.Write ($"0x{b:x02}"); - - rowFieldCounter++; - if (rowFieldCounter > MaxFieldsInRow) { - rowFieldCounter = 0; - outputWriter.WriteLine (); - } - } - - void HeaderFormatError (string whatsWrong) - { - throw new InvalidOperationException ($"Java.Interop.Tools.JavaCallableWrappers.TypeNameMapGenerator header format changed: {whatsWrong}"); - } - - int GetHeaderInteger (string name, string value) - { - int ret; - if (!Int32.TryParse (value, out ret)) - HeaderFormatError ($"failed to parse integer value from '{value}' for field '{name}'"); - return ret; - } - } - }; -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblyGenerator.cs index 17bfca35db7..ce92fdb8f48 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeAssemblyGenerator.cs @@ -12,34 +12,41 @@ abstract class NativeAssemblyGenerator protected string Indent { get; } = "\t"; protected NativeAssemblerTargetProvider TargetProvider { get; } + protected string TypemapsIncludeFile { get; } + protected string SharedIncludeFile { get; } + public string MainSourceFile { get; } - protected NativeAssemblyGenerator (NativeAssemblerTargetProvider targetProvider) + protected NativeAssemblyGenerator (NativeAssemblerTargetProvider targetProvider, string baseFilePath) { if (targetProvider == null) throw new ArgumentNullException (nameof (targetProvider)); + if (String.IsNullOrEmpty (baseFilePath)) + throw new ArgumentException ("must not be null or empty", nameof (baseFilePath)); + TargetProvider = targetProvider; + SharedIncludeFile = $"{baseFilePath}.shared.inc"; + TypemapsIncludeFile = $"{baseFilePath}.{targetProvider.AbiName}-managed.inc"; + MainSourceFile = $"{baseFilePath}.{targetProvider.AbiName}.s"; } - public void Write (StreamWriter output, string outputFileName) + public void Write (StreamWriter output) { if (output == null) throw new ArgumentNullException (nameof (output)); - WriteFileHeader (output, outputFileName); + WriteFileHeader (output); WriteSymbols (output); + WriteFileFooter (output); output.Flush (); } protected virtual void WriteSymbols (StreamWriter output) - { - } + {} protected void WriteEndLine (StreamWriter output, string comment = null, bool indent = true) { if (!String.IsNullOrEmpty (comment)) { - if (indent) - output.Write (Indent); - WriteComment (output, comment); + WriteComment (output, comment, indent); } output.WriteLine (); } @@ -51,16 +58,19 @@ protected void WriteComment (StreamWriter output, string comment, bool indent = protected void WriteCommentLine (StreamWriter output, string comment, bool indent = true) { - WriteComment (output, comment); + WriteComment (output, comment, indent); output.WriteLine (); } - protected virtual void WriteFileHeader (StreamWriter output, string outputFileName) + protected virtual void WriteFileHeader (StreamWriter output) { TargetProvider.WriteFileHeader (output, Indent); - output.WriteLine ($"{Indent}.file{Indent}\"{Path.GetFileName (outputFileName)}\""); + output.WriteLine ($"{Indent}.file{Indent}\"{Path.GetFileName (MainSourceFile)}\""); } + protected virtual void WriteFileFooter (StreamWriter output) + {} + protected virtual void WriteSection (StreamWriter output, string sectionName, bool hasStrings, bool writable) { output.Write ($"{Indent}.section{Indent}{sectionName},\"a"); // a - section is allocatable @@ -106,7 +116,19 @@ protected void WriteSymbol (StreamWriter output, string label, uint size, bool i WriteSymbol (output, null, null, label, size, alignBits: 0, isGlobal: isGlobal, isObject: isObject, alwaysWriteSize: alwaysWriteSize); } + protected void WriteSymbol (StreamWriter output, string label, uint size, uint alignBits, bool isGlobal, bool isObject, bool alwaysWriteSize) + { + WriteSymbol (output, null, null, label, size, alignBits, isGlobal: isGlobal, isObject: isObject, alwaysWriteSize: alwaysWriteSize); + } + protected void WriteSymbol (StreamWriter output, string symbolName, uint alignBits, uint fieldAlignBytes, bool isGlobal, bool alwaysWriteSize, Func structureWriter) + { + WriteStructureSymbol (output, symbolName, alignBits, isGlobal); + uint size = WriteStructure (output, fieldAlignBytes, structureWriter); + WriteStructureSize (output, symbolName, size, alwaysWriteSize); + } + + protected void WriteStructureSymbol (StreamWriter output, string symbolName, uint alignBits, bool isGlobal) { output.WriteLine ($"{Indent}.type{Indent}{symbolName}, {TargetProvider.TypePrefix}object"); if (alignBits > 0) @@ -115,14 +137,28 @@ protected void WriteSymbol (StreamWriter output, string symbolName, uint alignBi output.WriteLine ($"{Indent}.global{Indent}{symbolName}"); if (!String.IsNullOrEmpty (symbolName)) output.WriteLine ($"{symbolName}:"); + } + + protected void WriteStructureSize (StreamWriter output, string symbolName, uint size, bool alwaysWriteSize = false) + { + if (size == 0 && !alwaysWriteSize) + return; + if (String.IsNullOrEmpty (symbolName)) + throw new ArgumentException ("symbol name must be non-empty in order to write structure size", nameof (symbolName)); + + output.WriteLine ($"{Indent}.size{Indent}{symbolName}, {size}"); + } + + protected uint WriteStructure (StreamWriter output, uint fieldAlignBytes, Func structureWriter) + { writingStructure = true; structureByteCount = 0; structureAlignBytes = fieldAlignBytes; uint size = structureWriter != null ? structureWriter () : 0u; writingStructure = false; - if (alwaysWriteSize || size > 0) - output.WriteLine ($"{Indent}.size{Indent}{symbolName}, {size}"); + + return size; } protected virtual string QuoteValue (T value) @@ -136,13 +172,10 @@ protected void WriteSymbol (StreamWriter output, string symbolName, uint alignBi return $"{value}"; } - protected uint WritePointer (StreamWriter output, string targetName, string label = null, bool isGlobal = false) + protected uint WritePointer (StreamWriter output, string targetName = null, string label = null, bool isGlobal = false) { - if (String.IsNullOrEmpty (targetName)) - throw new ArgumentException ("must not be null or empty", nameof (targetName)); - - uint fieldSize = UpdateSize (output, targetName); - WriteSymbol (output, TargetProvider.PointerFieldType, targetName, label, size: 0, alignBits: 0, isGlobal: isGlobal, isObject: false, alwaysWriteSize: false); + uint fieldSize = UpdateSize (output, targetName ?? String.Empty); + WriteSymbol (output, TargetProvider.PointerFieldType, targetName ?? "0", label, size: 0, alignBits: 0, isGlobal: isGlobal, isObject: false, alwaysWriteSize: false); return fieldSize; } @@ -161,6 +194,9 @@ uint WriteDataPadding (StreamWriter output, uint nextFieldSize, uint sizeSoFar) protected uint WriteDataPadding (StreamWriter output, uint nbytes) { + if (nbytes == 0) + return 0; + output.WriteLine ($"{Indent}.zero{Indent}{nbytes}"); return nbytes; } @@ -173,24 +209,78 @@ protected uint WriteData (StreamWriter output, string value, string label, bool value = String.Empty; WriteSection (output, $".rodata.{label}", hasStrings: true, writable: false); - WriteSymbol (output, value, label, size: (ulong)(value.Length + 1), alignBits: 0, isGlobal: isGlobal, isObject: true, alwaysWriteSize: true); + WriteSymbol (output, value, isGlobal ? label : MakeLocalLabel (label), size: (ulong)(value.Length + 1), alignBits: 0, isGlobal: isGlobal, isObject: true, alwaysWriteSize: true); return TargetProvider.GetTypeSize (value); } + protected uint WriteAsciiData (StreamWriter output, string value, uint padToWidth = 0) + { + if (value == null) + value = String.Empty; + + uint size = (uint)value.Length; + if (size > 0) { + output.WriteLine ($"{Indent}.ascii{Indent}\"{value}\""); + } else if (padToWidth == 0) { + return WriteDataPadding (output, 1); + } + + if (padToWidth > size) + size += WriteDataPadding (output, padToWidth - size); + + return size; + } + + protected string MakeLocalLabel (string label) + { + return $".L.{label}"; + } + + uint UpdateSize (StreamWriter output, T[] value) + { + uint typeSize = TargetProvider.GetTypeSize (); + uint fieldSize = typeSize * (uint)value.Length; + + fieldSize += WriteDataPadding (output, fieldSize, structureByteCount); + structureByteCount += fieldSize; + + return fieldSize; + } + uint UpdateSize (StreamWriter output, T value) { uint fieldSize; - if (typeof (T) == typeof (string)) + Type t = typeof(T); + if (t == typeof (string)) fieldSize = TargetProvider.GetPointerSize (); else fieldSize = TargetProvider.GetTypeSize (value); + fieldSize += WriteDataPadding (output, fieldSize, structureByteCount); structureByteCount += fieldSize; return fieldSize; } + protected uint WriteData (StreamWriter output, byte[] value, string label = null, bool isGlobal = false) + { + uint fieldSize = UpdateSize (output, value); + var symbolValue = new StringBuilder (); + bool first = true; + foreach (byte b in value) { + if (!first) + symbolValue.Append (", "); + else + first = false; + + symbolValue.Append ($"0x{b:x02}"); + } + WriteSymbol (output, TargetProvider.MapType (), symbolValue.ToString (), symbolName: label, size: 0, alignBits: 0, isGlobal: isGlobal, isObject: false, alwaysWriteSize: false); + + return fieldSize; + } + protected uint WriteData (StreamWriter output, byte value, string label = null, bool isGlobal = false) { uint fieldSize = UpdateSize (output, value); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeTypeMappingData.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeTypeMappingData.cs new file mode 100644 index 00000000000..3ed37dee949 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeTypeMappingData.cs @@ -0,0 +1,44 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Xamarin.Android.Tasks +{ + class NativeTypeMappingData + { + public IDictionary Modules { get; } + public IDictionary AssemblyNames { get; } + public IDictionary JavaTypes { get; } + + public uint MapModuleCount { get; } + public uint JavaTypeCount { get; } + public uint JavaNameWidth { get; } + + public NativeTypeMappingData (IDictionary modules, int javaTypeCount, int javaNameWidth) + { + Modules = modules ?? throw new ArgumentNullException (nameof (modules)); + + MapModuleCount = (uint)modules.Count; + JavaTypeCount = (uint)javaTypeCount; + JavaNameWidth = (uint)javaNameWidth; + + AssemblyNames = new SortedDictionary (StringComparer.Ordinal); + JavaTypes = new SortedDictionary (StringComparer.Ordinal); + + List moduleList = modules.Values.ToList (); + int managedStringCounter = 0; + foreach (var kvp in modules) { + TypeMapGenerator.ModuleData data = kvp.Value; + data.AssemblyNameLabel = $"map_aname.{managedStringCounter++}"; + AssemblyNames.Add (data.AssemblyNameLabel, data.AssemblyName); + + int moduleIndex = moduleList.IndexOf (data); + foreach (var kvp2 in data.Types) { + TypeMapGenerator.TypeMapEntry entry = kvp2.Value; + entry.ModuleIndex = moduleIndex; + JavaTypes.Add (entry.JavaName, entry); + } + } + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs new file mode 100644 index 00000000000..012ab52189d --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -0,0 +1,386 @@ +using System; +using System.IO; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; + +using Mono.Cecil; + +using Java.Interop.Tools.Cecil; +using Java.Interop.Tools.JavaCallableWrappers; + +namespace Xamarin.Android.Tasks +{ + class TypeMapGenerator + { + const string TypeMapMagicString = "XATM"; // Xamarin Android TypeMap + const string TypeMapIndexMagicString = "XATI"; // Xamarin Android Typemap Index + const uint TypeMapFormatVersion = 1; // Keep in sync with the value in src/monodroid/jni/xamarin-app.hh + + sealed class UUIDByteArrayComparer : IComparer + { + public int Compare (byte[] left, byte[] right) + { + int ret; + + for (int i = 0; i < 16; i++) { + ret = left[i].CompareTo (right[i]); + if (ret != 0) + return ret; + } + + return 0; + } + } + + internal sealed class TypeMapEntry + { + public string JavaName; + public string ManagedTypeName; + public uint Token; + public int AssemblyNameIndex = -1; + public int ModuleIndex = -1; + } + + internal sealed class ModuleData + { + public Guid Mvid; + public AssemblyDefinition Assembly; + public SortedDictionary Types; + public Dictionary DuplicateTypes; + public string AssemblyName; + public string AssemblyNameLabel; + public string OutputFilePath; + } + + Action logger; + Encoding binaryEncoding; + byte[] moduleMagicString; + byte[] typemapIndexMagicString; + string[] supportedAbis; + + public IList GeneratedBinaryTypeMaps { get; } = new List (); + + public TypeMapGenerator (Action logger, string[] supportedAbis) + { + this.logger = logger ?? throw new ArgumentNullException (nameof (logger)); + if (supportedAbis == null) + throw new ArgumentNullException (nameof (supportedAbis)); + if (supportedAbis.Length == 0) + throw new ArgumentException ("must not be empty", nameof (supportedAbis)); + this.supportedAbis = supportedAbis; + + binaryEncoding = new UTF8Encoding (false); + moduleMagicString = binaryEncoding.GetBytes (TypeMapMagicString); + typemapIndexMagicString = binaryEncoding.GetBytes (TypeMapIndexMagicString); + } + + void LoggerShim (TraceLevel level, string message) + { + logger (message); + } + + public bool Generate (DirectoryAssemblyResolver resolver, IEnumerable assemblies, string outputDirectory, bool generateNativeAssembly) + { + if (assemblies == null) + throw new ArgumentNullException (nameof (assemblies)); + if (String.IsNullOrEmpty (outputDirectory)) + throw new ArgumentException ("must not be null or empty", nameof (outputDirectory)); + + if (!Directory.Exists (outputDirectory)) + Directory.CreateDirectory (outputDirectory); + + var scanner = new JavaTypeScanner (LoggerShim) { + ErrorOnCustomJavaObject = false + }; + + List javaTypes = scanner.GetJavaTypes (assemblies, resolver); + + int assemblyId = 0; + int maxJavaNameLength = 0; + int maxModuleFileNameLength = 0; + var knownAssemblies = new Dictionary (StringComparer.Ordinal); + var modules = new SortedDictionary (new UUIDByteArrayComparer ()); + Dictionary moduleCounter = null; + + foreach (TypeDefinition td in javaTypes) { + string assemblyName = td.Module.Assembly.FullName; + + if (!knownAssemblies.ContainsKey (assemblyName)) { + assemblyId++; + knownAssemblies.Add (assemblyName, assemblyId); + } + + byte[] moduleUUID = td.Module.Mvid.ToByteArray (); + ModuleData moduleData; + if (!modules.TryGetValue (moduleUUID, out moduleData)) { + if (moduleCounter == null) + moduleCounter = new Dictionary (); + + moduleData = new ModuleData { + Mvid = td.Module.Mvid, + Assembly = td.Module.Assembly, + AssemblyName = td.Module.Assembly.Name.Name, + Types = new SortedDictionary (StringComparer.Ordinal), + DuplicateTypes = new Dictionary (), + }; + modules.Add (moduleUUID, moduleData); + + if (!generateNativeAssembly) { + int moduleNum; + if (!moduleCounter.TryGetValue (moduleData.Assembly, out moduleNum)) { + moduleNum = 0; + moduleCounter [moduleData.Assembly] = 0; + } else { + moduleNum++; + moduleCounter [moduleData.Assembly] = moduleNum; + } + + string fileName = $"{moduleData.Assembly.Name.Name}.{moduleNum}.typemap"; + moduleData.OutputFilePath = Path.Combine (outputDirectory, fileName); + if (maxModuleFileNameLength < fileName.Length) + maxModuleFileNameLength = fileName.Length; + } + } + + string javaName = Java.Interop.Tools.TypeNameMappings.JavaNativeTypeManager.ToJniName (td); + if (generateNativeAssembly) { + if (javaName.Length > maxJavaNameLength) + maxJavaNameLength = javaName.Length; + } + + var entry = new TypeMapEntry { + JavaName = javaName, + ManagedTypeName = td.FullName, + Token = td.MetadataToken.ToUInt32 (), + AssemblyNameIndex = knownAssemblies [assemblyName] + }; + + if (moduleData.Types.ContainsKey (entry.JavaName)) { + logger ($"Warning: duplicate Java type name '{entry.JavaName}' (new token: {entry.Token})."); + moduleData.DuplicateTypes.Add (entry.Token, entry); + } else + moduleData.Types.Add (entry.JavaName, entry); + } + + NativeTypeMappingData data; + if (!generateNativeAssembly) { + string typeMapIndexPath = Path.Combine (outputDirectory, "typemap.index"); + using (var fs = File.OpenWrite (typeMapIndexPath)) { + using (var indexWriter = new BinaryWriter (fs)) { + OutputModules (outputDirectory, modules, indexWriter, maxModuleFileNameLength + 1); + indexWriter.Flush (); + } + } + GeneratedBinaryTypeMaps.Add (typeMapIndexPath); + + data = new NativeTypeMappingData (new Dictionary (), 0, 0); + } else { + data = new NativeTypeMappingData (modules, javaTypes.Count, maxJavaNameLength + 1); + } + + NativeAssemblerTargetProvider asmTargetProvider; + bool sharedBitsWritten = false; + foreach (string abi in supportedAbis) { + switch (abi.Trim ()) { + case "armeabi-v7a": + asmTargetProvider = new ARMNativeAssemblerTargetProvider (is64Bit: false); + break; + + case "arm64-v8a": + asmTargetProvider = new ARMNativeAssemblerTargetProvider (is64Bit: true); + break; + + case "x86": + asmTargetProvider = new X86NativeAssemblerTargetProvider (is64Bit: false); + break; + + case "x86_64": + asmTargetProvider = new X86NativeAssemblerTargetProvider (is64Bit: true); + break; + + default: + throw new InvalidOperationException ($"Unknown ABI {abi}"); + } + + var generator = new TypeMappingNativeAssemblyGenerator (asmTargetProvider, data, Path.Combine (outputDirectory, "typemaps"), sharedBitsWritten); + + using (var fs = File.OpenWrite (generator.MainSourceFile)) { + using (var sw = new StreamWriter (fs, new UTF8Encoding (false), bufferSize: 8192, leaveOpen: false)) { + generator.Write (sw); + sw.Flush (); + sharedBitsWritten = true; + } + } + } + return true; + } + + // Binary index file format, all data is little-endian: + // + // [Magic string] # XATI + // [Format version] # 32-bit integer, 4 bytes + // [Entry count] # 32-bit integer, 4 bytes + // [Module file name width] # 32-bit integer, 4 bytes + // [Index entries] # Format described below, [Entry count] entries + // + // Index entry format: + // + // [Module UUID][File name] + // + // Where: + // + // [Module UUID] is 16 bytes long + // [File name] is right-padded with characters to the [Module file name width] boundary. + // + void OutputModules (string outputDirectory, IDictionary modules, BinaryWriter indexWriter, int moduleFileNameWidth) + { + var moduleCounter = new Dictionary (); + + indexWriter.Write (typemapIndexMagicString); + indexWriter.Write (TypeMapFormatVersion); + indexWriter.Write (modules.Count); + indexWriter.Write (moduleFileNameWidth); + + foreach (var kvp in modules) { + byte[] mvid = kvp.Key; + ModuleData data = kvp.Value; + + OutputModule (outputDirectory, mvid, data, moduleCounter); + indexWriter.Write (mvid); + + string outputFilePath = Path.GetFileName (data.OutputFilePath); + indexWriter.Write (binaryEncoding.GetBytes (outputFilePath)); + PadField (indexWriter, outputFilePath.Length, moduleFileNameWidth); + } + } + + void OutputModule (string outputDirectory, byte[] moduleUUID, ModuleData moduleData, Dictionary moduleCounter) + { + if (moduleData.Types.Count == 0) + return; + + using (var fs = File.Open (moduleData.OutputFilePath, FileMode.Create)) { + using (var bw = new BinaryWriter (fs)) { + OutputModule (bw, moduleUUID, moduleData); + bw.Flush (); + } + } + GeneratedBinaryTypeMaps.Add (moduleData.OutputFilePath); + } + + // Binary file format, all data is little-endian: + // + // [Magic string] # XATM + // [Format version] # 32-bit integer, 4 bytes + // [Module UUID] # 16 bytes + // [Entry count] # 32-bit integer, 4 bytes + // [Duplicate count] # 32-bit integer, 4 bytes (might be 0) + // [Java type name width] # 32-bit integer, 4 bytes + // [Assembly name size] # 32-bit integer, 4 bytes + // [Assembly name] # Non-null terminated assembly name + // [Java-to-managed map] # Format described below, [Entry count] entries + // [Managed-to-java map] # Format described below, [Entry count] entries + // [Managed-to-java duplicates map] # Map of unique managed IDs which point to the same Java type name (might be empty) + // + // Java-to-managed map format: + // + // [Java type name][Managed type token ID] + // + // Each name is padded with to the width specified in the [Java type name width] field above. + // Names are written without the size prefix, instead they are always terminated with a nul character + // to make it easier and faster to handle by the native runtime. + // + // Each token ID is a 32-bit integer, 4 bytes + // + // + // Managed-to-java map format: + // + // [Managed type token ID][Java type name table index] + // + // Both fields are 32-bit integers, to a total of 8 bytes per entry. Index points into the + // [Java-to-managed map] table above. + // + // Managed-to-java duplicates map format: + // + // Format is identical to [Managed-to-java] above. + // + void OutputModule (BinaryWriter bw, byte[] moduleUUID, ModuleData moduleData) + { + bw.Write (moduleMagicString); + bw.Write (TypeMapFormatVersion); + bw.Write (moduleUUID); + + var javaNames = new SortedDictionary (StringComparer.Ordinal); + var managedTypes = new SortedDictionary (); + int maxJavaNameLength = 0; + + foreach (var kvp in moduleData.Types) { + TypeMapEntry entry = kvp.Value; + + javaNames.Add (entry.JavaName, entry.Token); + if (entry.JavaName.Length > maxJavaNameLength) + maxJavaNameLength = entry.JavaName.Length; + + managedTypes.Add (entry.Token, 0); + } + + var javaNameList = javaNames.Keys.ToList (); + foreach (var kvp in moduleData.Types) { + TypeMapEntry entry = kvp.Value; + var javaIndex = (uint)javaNameList.IndexOf (entry.JavaName); + managedTypes[entry.Token] = javaIndex; + } + + bw.Write (javaNames.Count); + bw.Write (moduleData.DuplicateTypes.Count); + bw.Write (maxJavaNameLength + 1); + + string assemblyName = moduleData.Assembly.Name.Name; + bw.Write (assemblyName.Length); + bw.Write (binaryEncoding.GetBytes (assemblyName)); + + foreach (var kvp in javaNames) { + string typeName = kvp.Key; + uint token = kvp.Value; + + bw.Write (binaryEncoding.GetBytes (typeName)); + PadField (bw, typeName.Length, maxJavaNameLength + 1); + bw.Write (token); + } + + WriteManagedTypes (managedTypes); + if (moduleData.DuplicateTypes.Count == 0) + return; + + var managedDuplicates = new SortedDictionary (); + foreach (var kvp in moduleData.DuplicateTypes) { + uint javaIndex = kvp.Key; + uint typeId = kvp.Value.Token; + + managedDuplicates.Add (javaIndex, typeId); + } + + WriteManagedTypes (managedDuplicates); + + void WriteManagedTypes (IDictionary types) + { + foreach (var kvp in types) { + uint token = kvp.Key; + uint javaIndex = kvp.Value; + + bw.Write (token); + bw.Write (javaIndex); + } + } + } + + void PadField (BinaryWriter bw, int valueWidth, int maxWidth) + { + for (int i = 0; i < maxWidth - valueWidth; i++) { + bw.Write ((byte)0); + } + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingNativeAssemblyGenerator.cs index 5e1c82f1c5a..7b87edb01c3 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingNativeAssemblyGenerator.cs @@ -1,75 +1,244 @@ using System; +using System.Collections.Generic; using System.IO; +using System.Linq; + using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks { class TypeMappingNativeAssemblyGenerator : NativeAssemblyGenerator { - NativeAssemblyDataStream dataStream; - string dataFileName; - uint dataSize; - string mappingFieldName; + readonly string baseFileName; + readonly NativeTypeMappingData mappingData; + readonly uint fieldAlignBytes; + readonly bool sharedBitsWritten; - public TypeMappingNativeAssemblyGenerator (NativeAssemblerTargetProvider targetProvider, NativeAssemblyDataStream dataStream, string dataFileName, uint dataSize, string mappingFieldName) - : base (targetProvider) + public TypeMappingNativeAssemblyGenerator (NativeAssemblerTargetProvider targetProvider, NativeTypeMappingData mappingData, string baseFileName, bool sharedBitsWritten) + : base (targetProvider, baseFileName) { - if (dataStream == null) - throw new ArgumentNullException (nameof (dataStream)); - if (String.IsNullOrEmpty (dataFileName)) - throw new ArgumentException ("must not be null or empty", nameof (dataFileName)); - if (String.IsNullOrEmpty (mappingFieldName)) - throw new ArgumentException ("must not be null or empty", nameof (mappingFieldName)); - - this.dataStream = dataStream; - this.dataFileName = dataFileName; - this.dataSize = dataSize; - this.mappingFieldName = mappingFieldName; - } + this.mappingData = mappingData ?? throw new ArgumentNullException (nameof (mappingData)); - public bool EmbedAssemblies { get; set; } + if (String.IsNullOrEmpty (baseFileName)) + throw new ArgumentException("must not be null or empty", nameof (baseFileName)); - protected override void WriteFileHeader (StreamWriter output, string outputFileName) + this.baseFileName = baseFileName; + fieldAlignBytes = targetProvider.Is64Bit ? 8u : 4u; + this.sharedBitsWritten = sharedBitsWritten; + } + + protected override void WriteFileFooter (StreamWriter output) { // The hash is written to make sure the assembly file which includes the data one is // actually different whenever the data changes. Relying on mapping header values for this // purpose would not be enough since the only change to the mapping might be a single-character // change in one of the type names and we must be sure the assembly is rebuilt in all cases, // thus the hash. - WriteEndLine (output, $"Data Hash: {Files.ToHexString (dataStream.GetStreamHash ())}", false); - base.WriteFileHeader (output, outputFileName); + WriteEndLine (output, $"Data Hash: TODO", indent: false); + base.WriteFileFooter (output); } protected override void WriteSymbols (StreamWriter output) { - WriteMappingHeader (output, dataStream, mappingFieldName); - WriteCommentLine (output, "Mapping data"); - WriteSymbol (output, mappingFieldName, dataSize, isGlobal: true, isObject: true, alwaysWriteSize: true); - if (EmbedAssemblies) { - output.WriteLine ($"{Indent}.include{Indent}\"{dataFileName}\""); + output.WriteLine (); + WriteHeaderField (output, "map_module_count", mappingData.MapModuleCount); + + output.WriteLine (); + WriteHeaderField (output, "java_type_count", mappingData.JavaTypeCount); + + output.WriteLine (); + WriteHeaderField (output, "java_name_width", mappingData.JavaNameWidth); + + bool haveAssemblyNames = mappingData.AssemblyNames.Count > 0; + bool haveModules = mappingData.Modules.Count > 0; + + output.WriteLine (); + if (haveAssemblyNames) { + output.WriteLine ($"{Indent}.include{Indent}\"{Path.GetFileName (SharedIncludeFile)}\""); + } else { + WriteCommentLine (output, $"No shared data present, {Path.GetFileName (SharedIncludeFile)} not generated"); + } + + if (haveModules) { + output.WriteLine ($"{Indent}.include{Indent}\"{Path.GetFileName (TypemapsIncludeFile)}\""); + } else { + WriteCommentLine (output, $"No modules defined, {Path.GetFileName (TypemapsIncludeFile)} not generated"); + } + output.WriteLine (); + + if (!sharedBitsWritten && haveAssemblyNames) { + using (var fs = File.Open (SharedIncludeFile, FileMode.Create)) { + using (var sharedOutput = new StreamWriter (fs, output.Encoding)) { + WriteAssemblyNames (sharedOutput); + sharedOutput.Flush (); + } + } + } + + if (haveModules) { + using (var fs = File.Open (TypemapsIncludeFile, FileMode.Create)) { + using (var mapOutput = new StreamWriter (fs, output.Encoding)) { + WriteMapModules (output, mapOutput, "map_modules"); + } + } + } else { + WriteMapModules (output, null, "map_modules"); + } + + WriteJavaMap (output, "map_java"); + } + + void WriteAssemblyNames (StreamWriter output) + { + foreach (var kvp in mappingData.AssemblyNames) { + string label = kvp.Key; + string name = kvp.Value; + + WriteData (output, name, label, isGlobal: false); + output.WriteLine (); + } + } + + void WriteManagedMaps (StreamWriter output, string moduleSymbolName, IDictionary entries) + { + if (entries == null || entries.Count == 0) + return; + + var javaTypes = mappingData.JavaTypes.Keys.ToList (); + var tokens = new SortedDictionary (); + foreach (var kvp in entries) { + TypeMapGenerator.TypeMapEntry entry = kvp.Value; + tokens[entry.Token] = (uint)javaTypes.IndexOf (entry.JavaName); + } + + WriteSection (output, $".rodata.{moduleSymbolName}", hasStrings: false, writable: false); + WriteStructureSymbol (output, moduleSymbolName, alignBits: 0, isGlobal: false); + + uint size = 0; + foreach (var kvp in tokens) { + uint token = kvp.Key; + uint javaMapIndex = kvp.Value; + + size += WriteStructure (output, fieldAlignBytes, () => WriteManagedMapEntry (output, token, javaMapIndex)); + } + + WriteStructureSize (output, moduleSymbolName, size); + output.WriteLine (); + } + + uint WriteManagedMapEntry (StreamWriter output, uint token, uint javaMapIndex) + { + uint size = WriteData (output, token); + size += WriteData (output, javaMapIndex); + + return size; + } + + void WriteMapModules (StreamWriter output, StreamWriter mapOutput, string symbolName) + { + WriteCommentLine (output, "Managed to Java map: START", indent: false); + WriteSection (output, $".data.rel.{symbolName}", hasStrings: false, writable: true); + WriteStructureSymbol (output, symbolName, alignBits: TargetProvider.MapModulesAlignBits, isGlobal: true); + + uint size = 0; + int moduleCounter = 0; + foreach (var kvp in mappingData.Modules) { + byte[] mvid = kvp.Key; + TypeMapGenerator.ModuleData data = kvp.Value; + + string mapName = $"module{moduleCounter++}_managed_to_java"; + string duplicateMapName; + + if (data.DuplicateTypes.Count == 0) + duplicateMapName = null; + else + duplicateMapName = $"{mapName}_duplicates"; + + size += WriteStructure (output, fieldAlignBytes, () => WriteMapModule (output, mapName, duplicateMapName, mvid, data)); + if (mapOutput != null) { + WriteManagedMaps (mapOutput, mapName, data.Types); + WriteManagedMaps (mapOutput, duplicateMapName, data.DuplicateTypes); + } } + + WriteStructureSize (output, symbolName, size); + WriteCommentLine (output, "Managed to Java map: END", indent: false); + output.WriteLine (); + } + + uint WriteMapModule (StreamWriter output, string mapName, string duplicateMapName, byte[] mvid, TypeMapGenerator.ModuleData data) + { + uint size = 0; + WriteCommentLine (output, $"module_uuid: {data.Mvid}"); + size += WriteData (output, mvid); + + WriteCommentLine (output, "entry_count"); + size += WriteData (output, data.Types.Count); + + WriteCommentLine (output, "duplicate_count"); + size += WriteData (output, data.DuplicateTypes.Count); + + WriteCommentLine (output, "map"); + size += WritePointer (output, mapName); + + WriteCommentLine (output, "duplicate_map"); + size += WritePointer (output, duplicateMapName); + + WriteCommentLine (output, $"assembly_name: {data.AssemblyName}"); + size += WritePointer (output, MakeLocalLabel (data.AssemblyNameLabel)); + + WriteCommentLine (output, "image"); + size += WritePointer (output); + + output.WriteLine (); + + return size; } - void WriteMappingHeader (StreamWriter output, NativeAssemblyDataStream dataStream, string mappingFieldName) + void WriteJavaMap (StreamWriter output, string symbolName) { + WriteCommentLine (output, "Java to managed map: START", indent: false); + WriteSection (output, $".rodata.{symbolName}", hasStrings: false, writable: false); + WriteStructureSymbol (output, symbolName, alignBits: TargetProvider.MapJavaAlignBits, isGlobal: true); + + uint size = 0; + int entryCount = 0; + foreach (var kvp in mappingData.JavaTypes) { + TypeMapGenerator.TypeMapEntry entry = kvp.Value; + size += WriteJavaMapEntry (output, entry, entryCount++); + } + + WriteStructureSize (output, symbolName, size); + WriteCommentLine (output, "Java to managed map: END", indent: false); output.WriteLine (); - WriteCommentLine (output, "Mapping header"); - WriteSection (output, $".data.{mappingFieldName}", hasStrings: false, writable: true); - WriteSymbol (output, $"{mappingFieldName}_header", alignBits: 2, fieldAlignBytes: 4, isGlobal: true, alwaysWriteSize: true, structureWriter: () => { - WriteCommentLine (output, "version"); - WriteData (output, dataStream.MapVersion); - - WriteCommentLine (output, "entry-count"); - WriteData (output, dataStream.MapEntryCount); - - WriteCommentLine (output, "entry-length"); - WriteData (output, dataStream.MapEntryLength); - - WriteCommentLine (output, "value-offset"); - WriteData (output, dataStream.MapValueOffset); - return 16; - }); + } + + uint WriteJavaMapEntry (StreamWriter output, TypeMapGenerator.TypeMapEntry entry, int entryIndex) + { + uint size = 0; + + WriteCommentLine (output, $"#{entryIndex}"); + WriteCommentLine (output, "module_index"); + size += WriteData (output, entry.ModuleIndex); + + WriteCommentLine (output, "type_token_id"); + size += WriteData (output, entry.Token); + + WriteCommentLine (output, "java_name"); + size += WriteAsciiData (output, entry.JavaName, mappingData.JavaNameWidth); + output.WriteLine (); + + return size; + } + + void WriteHeaderField (StreamWriter output, string name, uint value) + { + WriteCommentLine (output, $"{name}: START", indent: false); + WriteSection (output, $".rodata.{name}", hasStrings: false, writable: false); + WriteSymbol (output, name, size: 4, alignBits: 2, isGlobal: true, isObject: true, alwaysWriteSize: true); + WriteData (output, value); + WriteCommentLine (output, $"{name}: END", indent: false); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/X86NativeAssemblerTargetProvider.cs b/src/Xamarin.Android.Build.Tasks/Utilities/X86NativeAssemblerTargetProvider.cs index f6f2a33144a..bb36f000526 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/X86NativeAssemblerTargetProvider.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/X86NativeAssemblerTargetProvider.cs @@ -4,9 +4,15 @@ namespace Xamarin.Android.Tasks { class X86NativeAssemblerTargetProvider : NativeAssemblerTargetProvider { + const string X86 = "x86"; + const string X86_64 = "x86_64"; + public override bool Is64Bit { get; } public override string PointerFieldType { get; } public override string TypePrefix { get; } = "@"; + public override string AbiName => Is64Bit ? X86_64 : X86; + public override uint MapModulesAlignBits => Is64Bit ? 4u : 2u; + public override uint MapJavaAlignBits => Is64Bit ? 4u : 2u; public X86NativeAssemblerTargetProvider (bool is64Bit) { diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj index 75de516c1b7..ee908daff1e 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj @@ -39,6 +39,13 @@ + + ..\..\packages\System.Collections.Immutable.1.5.0\lib\netstandard2.0\System.Collections.Immutable.dll + + + + ..\..\packages\System.Reflection.Metadata.1.6.0\lib\netstandard2.0\System.Reflection.Metadata.dll + @@ -124,6 +131,7 @@ + @@ -171,6 +179,7 @@ + @@ -531,9 +540,9 @@ - + diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 0fbfcd922b7..8d94dfb02ca 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -70,6 +70,7 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. + @@ -1871,6 +1872,37 @@ because xbuild doesn't support framework reference assemblies. + + + + + + + + + + + + + + + + + + + - - - - - - - - <_GenerateJavaStubsDependsOnTargets> _SetLatestTargetFrameworkVersion; - _PrepareNativeAssemblySources; _PrepareAssemblies; + _GenerateTypeMaps; $(_AfterPrepareAssemblies); @@ -2096,7 +2116,6 @@ because xbuild doesn't support framework reference assemblies. - @@ -2218,6 +2237,7 @@ because xbuild doesn't support framework reference assemblies. EnablePreloadAssembliesDefault="$(_AndroidEnablePreloadAssembliesDefault)" PackageNamingPolicy="$(AndroidPackageNamingPolicy)" BoundExceptionType="$(AndroidBoundExceptionType)" + InstantRunEnabled="$(_InstantRunEnabled)" > diff --git a/src/monodroid/jni/android-system.cc b/src/monodroid/jni/android-system.cc index 068732f2e1d..56424557010 100644 --- a/src/monodroid/jni/android-system.cc +++ b/src/monodroid/jni/android-system.cc @@ -25,7 +25,7 @@ #include "monodroid.h" #include "monodroid-glue-internal.hh" #include "jni-wrappers.hh" -#include "xamarin-app.h" +#include "xamarin-app.hh" #include "cpp-util.hh" #if defined (DEBUG) || !defined (ANDROID) @@ -253,27 +253,6 @@ AndroidSystem::monodroid_get_system_property (const char *name, char **value) return len; } -size_t -AndroidSystem::monodroid_read_file_into_memory (const char *path, char *&value) -{ - size_t r = 0; - value = nullptr; - FILE *fp = utils.monodroid_fopen (path, "r"); - if (fp != nullptr) { - struct stat fileStat; - if (fstat (fileno (fp), &fileStat) == 0) { - r = ADD_WITH_OVERFLOW_CHECK (size_t, static_cast(fileStat.st_size), 1); - value = new char[r]; - size_t nread = fread (value, 1, static_cast(fileStat.st_size), fp); - if (nread == 0 || nread != static_cast(fileStat.st_size)) { - log_warn(LOG_DEFAULT, "While reading file %s: expected to read %u bytes, actually read %u bytes", path, r, nread); - } - } - fclose (fp); - } - return r; -} - #if defined (DEBUG) || !defined (ANDROID) size_t AndroidSystem::_monodroid_get_system_property_from_file (const char *path, char **value) diff --git a/src/monodroid/jni/android-system.hh b/src/monodroid/jni/android-system.hh index 7cb42c735ca..dd8580d3cbe 100644 --- a/src/monodroid/jni/android-system.hh +++ b/src/monodroid/jni/android-system.hh @@ -9,7 +9,7 @@ #include "util.hh" #include "cppcompat.hh" -#include "xamarin-app.h" +#include "xamarin-app.hh" #include "shared-constants.hh" #include "basic-android-system.hh" @@ -45,7 +45,6 @@ namespace xamarin::android::internal void create_update_dir (char *override_dir); int monodroid_get_system_property (const char *name, char **value); size_t monodroid_get_system_property_from_overrides (const char *name, char ** value); - size_t monodroid_read_file_into_memory (const char *path, char *&value); char* get_bundled_app (JNIEnv *env, jstring dir); int count_override_assemblies (); long get_gref_gc_threshold (); diff --git a/src/monodroid/jni/application_dso_stub.cc b/src/monodroid/jni/application_dso_stub.cc index 459bb600391..2cc3c860551 100644 --- a/src/monodroid/jni/application_dso_stub.cc +++ b/src/monodroid/jni/application_dso_stub.cc @@ -1,16 +1,16 @@ #include #include -#include "xamarin-app.h" +#include "xamarin-app.hh" -// This file MUST have valid values everywhere - the DSO it is compiled into is loaded by the +// This file MUST have "valid" values everywhere - the DSO it is compiled into is loaded by the // designer on desktop. +const uint32_t map_module_count = 0; +const uint32_t java_type_count = 0; +const uint32_t java_name_width = 0; -TypeMapHeader jm_typemap_header = { 1, 0, 0, 0 }; -uint8_t jm_typemap[] = { 0 }; - -TypeMapHeader mj_typemap_header = { 1, 0, 0, 0 }; -uint8_t mj_typemap[] = { 0 }; +const TypeMapModule map_modules[] = {}; +const TypeMapJava map_java[] = {}; ApplicationConfig application_config = { .uses_mono_llvm = false, diff --git a/src/monodroid/jni/embedded-assemblies-zip.cc b/src/monodroid/jni/embedded-assemblies-zip.cc index c254e85eaf5..9cef8b8ae29 100644 --- a/src/monodroid/jni/embedded-assemblies-zip.cc +++ b/src/monodroid/jni/embedded-assemblies-zip.cc @@ -74,19 +74,6 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, monodroid_sh if (compression_method != 0) continue; -#if defined (DEBUG) - if (utils.ends_with (file_name, ".jm")) { - md_mmap_info map_info = md_mmap_apk_file (fd, data_offset, file_size, file_name, apk_name); - add_type_mapping (&java_to_managed_maps, apk_name, file_name, (const char*)map_info.area); - continue; - } - - if (utils.ends_with (file_name, ".mj")) { - md_mmap_info map_info = md_mmap_apk_file (fd, data_offset, file_size, file_name, apk_name); - add_type_mapping (&managed_to_java_maps, apk_name, file_name, (const char*)map_info.area); - continue; - } -#endif if (strncmp (prefix, file_name, prefix_len) != 0) continue; diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index 9c703e8c70d..b7c9e9c70b2 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -22,7 +23,7 @@ #include "embedded-assemblies.hh" #include "globals.hh" #include "monodroid-glue.hh" -#include "xamarin-app.h" +#include "xamarin-app.hh" #include "cpp-util.hh" namespace xamarin::android::internal { @@ -125,114 +126,248 @@ EmbeddedAssemblies::TypeMappingInfo_compare_key (const void *a, const void *b) return strcmp (reinterpret_cast (a), reinterpret_cast (b)); } -inline const char* -EmbeddedAssemblies::find_entry_in_type_map (const char *name, uint8_t map[], TypeMapHeader& header) +template +const Entry* +EmbeddedAssemblies::binary_search (const Key *key, const Entry *base, size_t nmemb, [[maybe_unused]] size_t extra_size) { - const char *e = reinterpret_cast (bsearch (name, map, header.entry_count, header.entry_length, TypeMappingInfo_compare_key )); - if (e == nullptr) - return nullptr; - return e + header.value_offset; + static_assert (compare != nullptr, "compare is a required template parameter"); + + constexpr size_t size = sizeof(Entry); + while (nmemb > 0) { + const Entry *ret; + if constexpr (use_extra_size) { + ret = reinterpret_cast(reinterpret_cast(base) + (size + extra_size) * (nmemb / 2)); + } else { + ret = base + (nmemb / 2); + } + + int result = compare (key, ret); + if (result < 0) { + nmemb /= 2; + } else if (result > 0) { + if constexpr (use_extra_size) { + base = reinterpret_cast(reinterpret_cast(ret) + size + extra_size); + } else { + base = ret + 1; + } + nmemb -= nmemb / 2 + 1; + } else { + return ret; + } + } + + return nullptr; } -const char* -EmbeddedAssemblies::typemap_java_to_managed (const char *java) +MonoReflectionType* +EmbeddedAssemblies::typemap_java_to_managed (MonoString *java_type) { -#if defined (DEBUG) || !defined (ANDROID) - for (TypeMappingInfo *info = java_to_managed_maps; info != nullptr; info = info->next) { - /* log_warn (LOG_DEFAULT, "# jonp: checking file: %s!%s for type '%s'", info->source_apk, info->source_entry, java); */ - const char *e = reinterpret_cast (bsearch (java, info->mapping, static_cast(info->num_entries), static_cast(info->entry_length), TypeMappingInfo_compare_key)); - if (e == nullptr) - continue; - return e + info->value_offset; + timing_period total_time; + if (XA_UNLIKELY (utils.should_log (LOG_TIMING))) { + timing = new Timing (); + total_time.mark_start (); } + + simple_pointer_guard java_type_name (mono_string_to_utf8 (java_type)); + if (XA_UNLIKELY (!java_type_name || *java_type_name == '\0')) { + return nullptr; + } + + int32_t type_token_id = -1; + TypeMapModule *module; +#if defined (DEBUG) || !defined (ANDROID) + if (application_config.instant_run_enabled) { + size_t idx = 0; + for (; idx < module_count; idx++) { + const uint8_t *java_entry = binary_search (java_type_name.get (), modules[idx].java_map, modules[idx].entry_count, modules[idx].java_name_width + 3); + if (java_entry == nullptr) + continue; + type_token_id = *reinterpret_cast(java_entry + modules[idx].java_name_width); + break; + } + + if (idx >= module_count) { + log_error (LOG_ASSEMBLY, "typemap: unable to find module with Java type '%s' mapping", java_type_name.get ()); + return nullptr; + } + + module = &modules[idx]; + } else { #endif - return find_entry_in_type_map (java, jm_typemap, jm_typemap_header); -} + const TypeMapJava *java_entry = binary_search (java_type_name.get (), map_java, java_type_count, java_name_width); + if (java_entry == nullptr) { + log_warn (LOG_ASSEMBLY, "typemap: unable to find mapping to a managed type from Java type '%s'", java_type_name.get ()); + return nullptr; + } -const char* -EmbeddedAssemblies::typemap_managed_to_java (const char *managed) -{ + if (java_entry->module_index >= map_module_count) { + log_warn (LOG_ASSEMBLY, "typemap: mapping from Java type '%s' to managed type has invalid module index", java_type_name.get ()); + return nullptr; + } + + module = const_cast(&map_modules[java_entry->module_index]); + const TypeMapModuleEntry *entry = binary_search (&java_entry->type_token_id, module->map, module->entry_count); + if (entry == nullptr) { + log_warn (LOG_ASSEMBLY, "typemap: unable to find mapping from Java type '%s' to managed type with token ID %u in module [%s]", java_type_name.get (), java_entry->type_token_id, mono_guid_to_string (module->module_uuid)); + return nullptr; + } + type_token_id = java_entry->type_token_id; #if defined (DEBUG) || !defined (ANDROID) - for (TypeMappingInfo *info = managed_to_java_maps; info != nullptr; info = info->next) { - /* log_warn (LOG_DEFAULT, "# jonp: checking file: %s!%s for type '%s'", info->source_apk, info->source_entry, managed); */ - const char *e = reinterpret_cast (bsearch (managed, info->mapping, static_cast(info->num_entries), static_cast(info->entry_length), TypeMappingInfo_compare_key)); - if (e == nullptr) - continue; - return e + info->value_offset; } #endif - return find_entry_in_type_map (managed, mj_typemap, mj_typemap_header); -} -#if defined (DEBUG) || !defined (ANDROID) -void -EmbeddedAssemblies::extract_int (const char **header, const char *source_apk, const char *source_entry, const char *key_name, int *value) -{ - int read = 0; - int consumed = 0; - size_t key_name_len = 0; - char scanf_format [20] = { 0, }; + if (module->image == nullptr) { + module->image = mono_image_loaded (module->assembly_name); + if (module->image == nullptr) { + // TODO: load + log_error (LOG_ASSEMBLY, "typemap: assembly '%s' not loaded yet!", module->assembly_name); + } - if (header == nullptr || *header == nullptr) - return; + if (module->image == nullptr) { + log_error (LOG_ASSEMBLY, "typemap: unable to load assembly '%s' when looking up managed type corresponding to Java type '%s'", module->assembly_name, java_type_name.get ()); + return nullptr; + } + } - key_name_len = strlen (key_name); - if (key_name_len >= (sizeof (scanf_format) - sizeof ("=%d%n"))) { - *header = nullptr; - return; + log_debug (LOG_ASSEMBLY, "typemap: java type '%s' corresponds to managed token id %u (0x%x)", java_type_name.get (), type_token_id, type_token_id); + MonoClass *klass = mono_class_get (module->image, static_cast(type_token_id)); + if (klass == nullptr) { + log_error (LOG_ASSEMBLY, "typemap: unable to find managed type with token ID %u in assembly '%s', corresponding to Java type '%s'", type_token_id, module->assembly_name, java_type_name.get ()); + return nullptr; } - snprintf (scanf_format, sizeof (scanf_format), "%s=%%d%%n", key_name); + MonoReflectionType *ret = mono_type_get_object (mono_domain_get (), mono_class_get_type (klass)); + if (XA_UNLIKELY (utils.should_log (LOG_TIMING))) { + total_time.mark_end (); - read = sscanf (*header, scanf_format, value, &consumed); - if (read != 1) { - log_warn (LOG_DEFAULT, "Could not read header '%s' value from '%s!%s': read %i elements, expected 1 element. Contents: '%s'", - key_name, source_apk, source_entry, read, *header); - *header = nullptr; - return; + Timing::info (total_time, "Typemap.java_to_managed: end, total time"); } - *header = *header + consumed + 1; + + return ret; } -bool -EmbeddedAssemblies::add_type_mapping (TypeMappingInfo **info, const char *source_apk, const char *source_entry, const char *addr) +int +EmbeddedAssemblies::compare_java_name (const char *java_name, const TypeMapJava *entry) { - TypeMappingInfo *p = new TypeMappingInfo (); // calloc (1, sizeof (struct TypeMappingInfo)); - int version = 0; - const char *data = addr; - - extract_int (&data, source_apk, source_entry, "version", &version); - if (version != 1) { - delete p; - log_warn (LOG_DEFAULT, "Unsupported version '%i' within type mapping file '%s!%s'. Ignoring...", version, source_apk, source_entry); - return false; + return strcmp (java_name, reinterpret_cast(entry->java_name)); +} + +#if defined (DEBUG) || !defined (ANDROID) +int +EmbeddedAssemblies::compare_java_name (const char *java_name, const uint8_t *entry) +{ + return strcmp (java_name, reinterpret_cast(entry)); +} +#endif // DEBUG || !ANDROID + +const char* +EmbeddedAssemblies::typemap_managed_to_java (const uint8_t *mvid, const int32_t token) +{ + timing_period total_time; + if (XA_UNLIKELY (utils.should_log (LOG_TIMING))) { + timing = new Timing (); + total_time.mark_start (); } - extract_int (&data, source_apk, source_entry, "entry-count", &p->num_entries); - extract_int (&data, source_apk, source_entry, "entry-len", &p->entry_length); - extract_int (&data, source_apk, source_entry, "value-offset", &p->value_offset); - p->mapping = data; + if (mvid == nullptr) { + return nullptr; + } - if ((p->mapping == 0) || - (p->num_entries <= 0) || - (p->entry_length <= 0) || - (p->value_offset >= p->entry_length) || - (p->mapping == nullptr)) { - log_warn (LOG_DEFAULT, "Could not read type mapping file '%s!%s'. Ignoring...", source_apk, source_entry); - delete p; - return false; + const TypeMapModule *map; + uint32_t map_entry_count; +#if defined (DEBUG) || !defined (ANDROID) + if (application_config.instant_run_enabled) { + map = modules; + map_entry_count = module_count; + } else { +#endif + map = map_modules; + map_entry_count = map_module_count; +#if defined (DEBUG) || !defined (ANDROID) + } +#endif + const TypeMapModule *match = binary_search (mvid, map, map_entry_count); + if (match == nullptr) { + log_warn (LOG_ASSEMBLY, "typemap: module matching MVID [%s] not found.", mono_guid_to_string (mvid)); + return nullptr; } - p->source_apk = strdup (source_apk); - p->source_entry = strdup (source_entry); - if (*info) { - (*info)->next = p; + if (match->map == nullptr) { + log_warn (LOG_ASSEMBLY, "typemap: module with MVID [%s] has no associated type map.", mono_guid_to_string (mvid)); + return nullptr; + } + + // Each map entry is a pair of 32-bit integers: [TypeTokenID][JavaMapArrayIndex] + const TypeMapModuleEntry *entry = binary_search (&token, match->map, match->entry_count); + if (entry == nullptr) { + if (match->duplicate_count > 0 && match->duplicate_map != nullptr) { + log_debug (LOG_ASSEMBLY, "typemap: searching module [%s] duplicate map for token %u (0x%x)", mono_guid_to_string (mvid), token, token); + entry = binary_search (&token, match->duplicate_map, match->duplicate_count); + } + + if (entry == nullptr) { + log_warn (LOG_ASSEMBLY, "typemap: type with token %d (0x%x) in module {%s} (%s) not found.", token, token, mono_guid_to_string (mvid), match->assembly_name); + return nullptr; + } + } + + uint32_t java_entry_count; +#if defined (DEBUG) || !defined (ANDROID) + if (application_config.instant_run_enabled) { + java_entry_count = match->entry_count; } else { - *info = p; +#endif + java_entry_count = java_type_count; +#if defined (DEBUG) || !defined (ANDROID) } - return true; +#endif + if (entry->java_map_index >= java_entry_count) { + log_warn (LOG_ASSEMBLY, "typemap: type with token %d (0x%x) in module {%s} (%s) has invalid Java type index %u", token, token, mono_guid_to_string (mvid), match->assembly_name, entry->java_map_index); + return nullptr; + } + + char *ret; +#if defined (DEBUG) || !defined (ANDROID) + if (application_config.instant_run_enabled) { + ret = reinterpret_cast(match->java_map + ((match->java_name_width + 4) * entry->java_map_index)); + } else { +#endif + const TypeMapJava *java_entry = reinterpret_cast (reinterpret_cast(map_java) + ((sizeof(TypeMapJava) + java_name_width) * entry->java_map_index)); + ret = reinterpret_cast(const_cast(java_entry->java_name)); +#if defined (DEBUG) || !defined (ANDROID) + } +#endif + + if (XA_UNLIKELY (utils.should_log (LOG_TIMING))) { + total_time.mark_end (); + + Timing::info (total_time, "Typemap.managed_to_java: end, total time"); + } + + log_debug ( + LOG_ASSEMBLY, + "typemap: type with token %d (0x%x) in module {%s} (%s) corresponds to Java type '%s'", + token, + token, + mono_guid_to_string (mvid), + match->assembly_name, + ret + ); + + return ret; +} + +int +EmbeddedAssemblies::compare_type_token (const int32_t *token, const TypeMapModuleEntry *entry) +{ + return *token - entry->type_token_id; +} + +int +EmbeddedAssemblies::compare_mvid (const uint8_t *mvid, const TypeMapModule *module) +{ + return memcmp (mvid, module->module_uuid, 16); } -#endif // DEBUG || !ANDROID EmbeddedAssemblies::md_mmap_info EmbeddedAssemblies::md_mmap_apk_file (int fd, uint32_t offset, uint32_t size, const char* filename, const char* apk) @@ -299,50 +434,260 @@ EmbeddedAssemblies::gather_bundled_assemblies_from_apk (const char* apk, monodro } #if defined (DEBUG) || !defined (ANDROID) +template +bool +EmbeddedAssemblies::typemap_read_header ([[maybe_unused]] int dir_fd, const char *file_type, const char *dir_path, const char *file_path, uint32_t expected_magic, H &header, size_t &file_size, int &fd) +{ + struct stat sbuf; + int res; + +#if __ANDROID_API__ < 21 + simple_pointer_guard full_file_path = utils.path_combine (dir_path, file_path); + res = stat (full_file_path, &sbuf); +#else + res = fstatat (dir_fd, file_path, &sbuf, 0); +#endif + if (res < 0) { + log_error (LOG_ASSEMBLY, "typemap: failed to stat %s file '%s/%s': %s", file_type, dir_path, file_path, strerror (errno)); + return false; + } + + file_size = static_cast(sbuf.st_size); + if (file_size < sizeof (header)) { + log_error (LOG_ASSEMBLY, "typemap: %s file '%s/%s' is too small (must be at least %u bytes)", file_type, dir_path, file_path, sizeof (header)); + return false; + } + +#if __ANDROID_API__ < 21 + fd = open (full_file_path, O_RDONLY); +#else + fd = openat (dir_fd, file_path, O_RDONLY); +#endif + if (fd < 0) { + log_error (LOG_ASSEMBLY, "typemap: failed to open %s file %s/%s for reading: %s", file_type, dir_path, file_path, strerror (errno)); + return false; + } + + ssize_t nread = read (fd, &header, sizeof (header)); + if (nread <= 0) { + if (nread < 0) { + log_error (LOG_ASSEMBLY, "typemap: failed to read %s file header from '%s/%s': %s", file_type, dir_path, file_path, strerror (errno)); + } else { + log_error (LOG_ASSEMBLY, "typemap: end of file while reading %s file header from '%s/%s'", file_type, dir_path, file_path); + } + + return false; + } + + if (header.magic != expected_magic) { + log_error (LOG_ASSEMBLY, "typemap: invalid magic value in the %s file header from '%s/%s': expected 0x%X, got 0x%X", file_type, dir_path, file_path, expected_magic, header.magic); + return false; + } + + if (header.version != MODULE_FORMAT_VERSION) { + log_error (LOG_ASSEMBLY, "typemap: incompatible %s format version. This build supports only version %u, file '%s/%s' uses version %u", file_type, MODULE_FORMAT_VERSION, dir_path, file_path, header.version); + return false; + } + + return true; +} + +uint8_t* +EmbeddedAssemblies::typemap_load_index (TypeMapIndexHeader &header, size_t file_size, int index_fd) +{ + constexpr size_t UUID_SIZE = 16; + + size_t entry_size = header.module_file_name_width + UUID_SIZE; + size_t data_size = entry_size * module_count; + if (sizeof(header) + data_size > file_size) { + log_error (LOG_ASSEMBLY, "typemap: index file is too small, expected %u, found %u bytes", data_size + sizeof(header), file_size); + return nullptr; + } + + auto data = new uint8_t [data_size]; + ssize_t nread = read (index_fd, data, data_size); + if (nread != static_cast(data_size)) { + log_error (LOG_ASSEMBLY, "typemap: failed to read %u bytes from index file. %s", data_size, strerror (errno)); + return nullptr; + } + + uint8_t *p = data; + for (size_t i = 0; i < module_count; i++) { + memcpy (modules[i].module_uuid, p, UUID_SIZE); + modules[i].assembly_name = reinterpret_cast(p + UUID_SIZE); + p += entry_size; + } + + return data; +} + +uint8_t* +EmbeddedAssemblies::typemap_load_index (int dir_fd, const char *dir_path, const char *index_path) +{ + log_debug (LOG_ASSEMBLY, "typemap: loading TypeMap index file '%s/%s'", dir_path, index_path); + + TypeMapIndexHeader header; + size_t file_size; + int fd = -1; + uint8_t *data = nullptr; + + if (!typemap_read_header (dir_fd, "TypeMap index", dir_path, index_path, MODULE_INDEX_MAGIC, header, file_size, fd)) { + goto cleanup; + } + + module_count = header.entry_count; + modules = new TypeMapModule[module_count](); + data = typemap_load_index (header, file_size, fd); + + cleanup: + if (fd >= 0) + close (fd); + + return data; +} + +bool +EmbeddedAssemblies::typemap_load_file (BinaryTypeMapHeader &header, const char *dir_path, const char *file_path, int file_fd, TypeMapModule &module) +{ + size_t alloc_size = ADD_WITH_OVERFLOW_CHECK (size_t, header.assembly_name_length, 1); + module.assembly_name = new char[alloc_size]; + + ssize_t nread = read (file_fd, module.assembly_name, header.assembly_name_length); + module.assembly_name [header.assembly_name_length] = 0; + module.entry_count = header.entry_count; + + log_debug ( + LOG_ASSEMBLY, + "typemap: '%s/%s':: entry count == %u; duplicate entry count == %u; Java type name field width == %u; MVID == %s; assembly name length == %u; assembly name == %s", + dir_path, file_path, header.entry_count, header.duplicate_count, header.java_name_width, + mono_guid_to_string (header.module_uuid), header.assembly_name_length, module.assembly_name + ); + + alloc_size = MULTIPLY_WITH_OVERFLOW_CHECK (size_t, header.java_name_width + 4, header.entry_count); + module.java_name_width = header.java_name_width; + module.java_map = new uint8_t[alloc_size]; + nread = read (file_fd, module.java_map, alloc_size); + if (nread != static_cast(alloc_size)) { + log_error (LOG_ASSEMBLY, "typemap: failed to read %u bytes (java-to-managed) from module file %s/%s. %s", alloc_size, dir_path, file_path, strerror (errno)); + return false; + } + + module.map = new TypeMapModuleEntry[header.entry_count]; + alloc_size = MULTIPLY_WITH_OVERFLOW_CHECK (size_t, sizeof(TypeMapModuleEntry), header.entry_count); + nread = read (file_fd, module.map, alloc_size); + if (nread != static_cast(alloc_size)) { + log_error (LOG_ASSEMBLY, "typemap: failed to read %u bytes (managed-to-java) from module file %s/%s. %s", alloc_size, dir_path, file_path, strerror (errno)); + return false; + } + + // alloc_size = module.java_name_width + 1; + // auto chars = new char[alloc_size](); + // uint8_t *p = module.java_map; + // log_debug (LOG_ASSEMBLY, "Java entries in %s/%s", dir_path, file_path); + // for (size_t i = 0; i < module.entry_count; i++) { + // memcpy (chars, p, module.java_name_width); + // uint32_t token = *reinterpret_cast(p + module.java_name_width); + // log_debug (LOG_ASSEMBLY, " %04u: %s; %u (0x%x)", i, chars, token, token); + // p += module.java_name_width + 4; + // } + // delete[] chars; + + // log_debug (LOG_ASSEMBLY, "Managed entries in %s/%s", dir_path, file_path); + // for (size_t i = 0; i < module.entry_count; i++) { + // log_debug (LOG_ASSEMBLY, " %04u: token %u (0x%x); index %u", i, module.map[i].type_token_id, module.map[i].type_token_id, module.map[i].java_map_index); + // } + + if (header.duplicate_count == 0) + return true; + + module.duplicate_map = new TypeMapModuleEntry[header.duplicate_count]; + alloc_size = MULTIPLY_WITH_OVERFLOW_CHECK (size_t, sizeof(TypeMapModuleEntry), header.duplicate_count); + nread = read (file_fd, module.duplicate_map, alloc_size); + if (nread != static_cast(alloc_size)) { + log_error (LOG_ASSEMBLY, "typemap: failed to read %u bytes (managed-to-java duplicates) from module file %s/%s. %s", alloc_size, dir_path, file_path, strerror (errno)); + return false; + } + + return true; +} + +bool +EmbeddedAssemblies::typemap_load_file (int dir_fd, const char *dir_path, const char *file_path, TypeMapModule &module) +{ + log_debug (LOG_ASSEMBLY, "typemap: loading TypeMap file '%s/%s'", dir_path, file_path); + + bool ret = true; + BinaryTypeMapHeader header; + size_t file_size; + int fd = -1; + + module.java_map = nullptr; + module.map = nullptr; + module.duplicate_map = nullptr; + + if (!typemap_read_header (dir_fd, "TypeMap", dir_path, file_path, MODULE_MAGIC, header, file_size, fd)) { + ret = false; + goto cleanup; + } + + ret = typemap_load_file (header, dir_path, file_path, fd, module); + + cleanup: + if (fd >= 0) + close (fd); + + if (!ret) { + delete[] module.java_map; + module.java_map = nullptr; + delete[] module.map; + module.map = nullptr; + delete[] module.duplicate_map; + module.duplicate_map = nullptr; + } + + return ret; +} + void EmbeddedAssemblies::try_load_typemaps_from_directory (const char *path) { - // read the entire typemap file into a string - // process the string using the add_type_mapping - char *dir_path = utils.path_combine (path, "typemaps"); - if (dir_path == nullptr || !utils.directory_exists (dir_path)) { - log_warn (LOG_DEFAULT, "directory does not exist: `%s`", dir_path); - free (dir_path); + if (!application_config.instant_run_enabled) { + log_info (LOG_ASSEMBLY, "typemap: instant run disabled, not loading type maps from storage"); return; } + simple_pointer_guard dir_path (utils.path_combine (path, "typemaps")); monodroid_dir_t *dir; if ((dir = utils.monodroid_opendir (dir_path)) == nullptr) { - log_warn (LOG_DEFAULT, "could not open directory: `%s`", dir_path); - free (dir_path); + log_warn (LOG_DEFAULT, "typemap: could not open directory: `%s`", dir_path.get ()); return; } - monodroid_dirent_t *e; - while ((e = androidSystem.readdir (dir)) != nullptr) { -#if WINDOWS - char *file_name = utils.utf16_to_utf8 (e->d_name); -#else /* def WINDOWS */ - char *file_name = e->d_name; -#endif /* ndef WINDOWS */ - char *file_path = utils.path_combine (dir_path, file_name); - if (utils.monodroid_dirent_hasextension (e, ".mj") || utils.monodroid_dirent_hasextension (e, ".jm")) { - char *val = nullptr; - size_t len = androidSystem.monodroid_read_file_into_memory (file_path, val); - if (len > 0 && val != nullptr) { - if (utils.monodroid_dirent_hasextension (e, ".mj")) { - if (!add_type_mapping (&managed_to_java_maps, file_path, override_typemap_entry_name, ((const char*)val))) - delete[] val; - } else if (utils.monodroid_dirent_hasextension (e, ".jm")) { - if (!add_type_mapping (&java_to_managed_maps, file_path, override_typemap_entry_name, ((const char*)val))) - delete[] val; - } - } + int dir_fd; +#if __ANDROID_API__ < 21 + dir_fd = -1; +#else + dir_fd = dirfd (dir); +#endif + + constexpr char index_name[] = "typemap.index"; + + // The pointer must be stored here because, after index is loaded, module.assembly_name points into the index data + // and must be valid until after the actual module file is loaded. + simple_pointer_guard index_data = typemap_load_index (dir_fd, dir_path, index_name); + if (!index_data) { + log_fatal (LOG_ASSEMBLY, "typemap: unable to load TypeMap data index from '%s/%s'", dir_path.get (), index_name); + exit (FATAL_EXIT_NO_ASSEMBLIES); // TODO: use a new error code here + } + + for (size_t i = 0; i < module_count; i++) { + TypeMapModule &module = modules[i]; + if (!typemap_load_file (dir_fd, dir_path, module.assembly_name, module)) { + continue; } } + utils.monodroid_closedir (dir); - free (dir_path); - return; } #endif diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index 1f3a024c960..37f167227a9 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -3,8 +3,11 @@ #define INC_MONODROID_EMBEDDED_ASSEMBLIES_H #include +#include #include +#include "xamarin-app.hh" + struct TypeMapHeader; namespace xamarin::android::internal { @@ -39,8 +42,8 @@ namespace xamarin::android::internal { void try_load_typemaps_from_directory (const char *path); #endif void install_preload_hooks (); - const char* typemap_java_to_managed (const char *java); - const char* typemap_managed_to_java (const char *managed); + MonoReflectionType* typemap_java_to_managed (MonoString *java_type); + const char* typemap_managed_to_java (const uint8_t *mvid, const int32_t token); /* returns current number of *all* assemblies found from all invocations */ template @@ -66,9 +69,13 @@ namespace xamarin::android::internal { size_t register_from (const char *apk_file, monodroid_should_register should_register); void gather_bundled_assemblies_from_apk (const char* apk, monodroid_should_register should_register); MonoAssembly* open_from_bundles (MonoAssemblyName* aname, bool ref_only); - void extract_int (const char **header, const char *source_apk, const char *source_entry, const char *key_name, int *value); #if defined (DEBUG) || !defined (ANDROID) - bool add_type_mapping (TypeMappingInfo **info, const char *source_apk, const char *source_entry, const char *addr); + template + bool typemap_read_header (int dir_fd, const char *file_type, const char *dir_path, const char *file_path, uint32_t expected_magic, H &header, size_t &file_size, int &fd); + uint8_t* typemap_load_index (int dir_fd, const char *dir_path, const char *index_path); + uint8_t* typemap_load_index (TypeMapIndexHeader &header, size_t file_size, int index_fd); + bool typemap_load_file (int dir_fd, const char *dir_path, const char *file_path, TypeMapModule &module); + bool typemap_load_file (BinaryTypeMapHeader &header, const char *dir_path, const char *file_path, int file_fd, TypeMapModule &module); #endif // DEBUG || !ANDROID bool register_debug_symbols_for_assembly (const char *entry_name, MonoBundledAssembly *assembly, const mono_byte *debug_contents, int debug_size); @@ -77,7 +84,6 @@ namespace xamarin::android::internal { static MonoAssembly* open_from_bundles_full (MonoAssemblyName *aname, char **assemblies_path, void *user_data); static MonoAssembly* open_from_bundles_refonly (MonoAssemblyName *aname, char **assemblies_path, void *user_data); static int TypeMappingInfo_compare_key (const void *a, const void *b); - const char *find_entry_in_type_map (const char *name, uint8_t map[], TypeMapHeader& header); void zip_load_entries (int fd, const char *apk_name, monodroid_should_register should_register); bool zip_read_cd_info (int fd, uint32_t& cd_offset, uint32_t& cd_size, uint16_t& cd_entries); @@ -95,6 +101,16 @@ namespace xamarin::android::internal { return assemblies_prefix_override != nullptr ? assemblies_prefix_override : assemblies_prefix; } + template + const Entry* binary_search (const Key *key, const Entry *base, size_t nmemb, size_t extra_size = 0); + + static int compare_mvid (const uint8_t *mvid, const TypeMapModule *module); + static int compare_type_token (const int32_t *token, const TypeMapModuleEntry *entry); + static int compare_java_name (const char *java_name, const TypeMapJava *entry); +#if defined (DEBUG) || !defined (ANDROID) + static int compare_java_name (const char *java_name, const uint8_t *java_map); +#endif + private: bool register_debug_symbols; MonoBundledAssembly **bundled_assemblies = nullptr; @@ -102,6 +118,8 @@ namespace xamarin::android::internal { #if defined (DEBUG) || !defined (ANDROID) TypeMappingInfo *java_to_managed_maps; TypeMappingInfo *managed_to_java_maps; + TypeMapModule *modules; + size_t module_count; #endif // DEBUG || !ANDROID const char *assemblies_prefix_override = nullptr; }; diff --git a/src/monodroid/jni/external-api.cc b/src/monodroid/jni/external-api.cc index 63ae892c1cc..aa58b950ca4 100644 --- a/src/monodroid/jni/external-api.cc +++ b/src/monodroid/jni/external-api.cc @@ -159,15 +159,9 @@ _monodroid_get_display_dpi (float *x_dpi, float *y_dpi) } MONO_API const char * -monodroid_typemap_java_to_managed (const char *java) +monodroid_typemap_managed_to_java (const uint8_t *mvid, const int32_t token) { - return embeddedAssemblies.typemap_java_to_managed (java); -} - -MONO_API const char * -monodroid_typemap_managed_to_java (const char *managed) -{ - return embeddedAssemblies.typemap_managed_to_java (managed); + return embeddedAssemblies.typemap_managed_to_java (mvid, token); } MONO_API int monodroid_embedded_assemblies_set_assemblies_prefix (const char *prefix) diff --git a/src/monodroid/jni/monodroid-glue-internal.hh b/src/monodroid/jni/monodroid-glue-internal.hh index 942426640b3..c4d373c129e 100644 --- a/src/monodroid/jni/monodroid-glue-internal.hh +++ b/src/monodroid/jni/monodroid-glue-internal.hh @@ -176,6 +176,7 @@ namespace xamarin::android::internal static void jit_done (MonoProfiler *prof, MonoMethod *method, MonoJitInfo* jinfo); static void thread_start (MonoProfiler *prof, uintptr_t tid); static void thread_end (MonoProfiler *prof, uintptr_t tid); + static MonoReflectionType* typemap_java_to_managed (MonoString *java_type_name); #if defined (DEBUG) void set_debug_env_vars (void); diff --git a/src/monodroid/jni/monodroid-glue.cc b/src/monodroid/jni/monodroid-glue.cc index 2d0e740cb27..b1ebef9fad2 100644 --- a/src/monodroid/jni/monodroid-glue.cc +++ b/src/monodroid/jni/monodroid-glue.cc @@ -68,7 +68,7 @@ #include "mkbundle-api.h" #include "monodroid-glue-internal.hh" #include "globals.hh" -#include "xamarin-app.h" +#include "xamarin-app.hh" #include "timing.hh" #ifndef WINDOWS @@ -345,12 +345,14 @@ inline void MonodroidRuntime::gather_bundled_assemblies (JNIEnv *env, jstring_array_wrapper &runtimeApks, bool register_debug_symbols, size_t *out_user_assemblies_count) { #if defined(DEBUG) || !defined (ANDROID) - for (size_t i = 0; i < AndroidSystem::MAX_OVERRIDES; ++i) { - const char *p = androidSystem.get_override_dir (i); - if (!utils.directory_exists (p)) - continue; - log_info (LOG_ASSEMBLY, "Loading TypeMaps from %s", p); - embeddedAssemblies.try_load_typemaps_from_directory (p); + if (application_config.instant_run_enabled) { + for (size_t i = 0; i < AndroidSystem::MAX_OVERRIDES; ++i) { + const char *p = androidSystem.get_override_dir (i); + if (!utils.directory_exists (p)) + continue; + log_info (LOG_ASSEMBLY, "Loading TypeMaps from %s", p); + embeddedAssemblies.try_load_typemaps_from_directory (p); + } } #endif @@ -918,6 +920,8 @@ MonodroidRuntime::lookup_bridge_info (MonoDomain *domain, MonoImage *image, cons void MonodroidRuntime::init_android_runtime (MonoDomain *domain, JNIEnv *env, jclass runtimeClass, jobject loader) { + mono_add_internal_call ("Java.Interop.TypeManager::monodroid_typemap_java_to_managed", reinterpret_cast(typemap_java_to_managed)); + struct JnienvInitializeArgs init = {}; init.javaVm = osBridge.get_jvm (); init.env = env; @@ -1399,6 +1403,12 @@ MonodroidRuntime::create_and_initialize_domain (JNIEnv* env, jclass runtimeClass return domain; } +MonoReflectionType* +MonodroidRuntime::typemap_java_to_managed (MonoString *java_type_name) +{ + return embeddedAssemblies.typemap_java_to_managed (java_type_name); +} + inline void MonodroidRuntime::Java_mono_android_Runtime_initInternal (JNIEnv *env, jclass klass, jstring lang, jobjectArray runtimeApksJava, jstring runtimeNativeLibDir, jobjectArray appDirs, jobject loader, diff --git a/src/monodroid/jni/xamarin-app.h b/src/monodroid/jni/xamarin-app.h deleted file mode 100644 index 96812303a4e..00000000000 --- a/src/monodroid/jni/xamarin-app.h +++ /dev/null @@ -1,42 +0,0 @@ -// Dear Emacs, this is a -*- C++ -*- header -#ifndef __XAMARIN_ANDROID_TYPEMAP_H -#define __XAMARIN_ANDROID_TYPEMAP_H - -#include - -#include "monodroid.h" - -struct TypeMapHeader -{ - uint32_t version; - uint32_t entry_count; - uint32_t entry_length; - uint32_t value_offset; -}; - -struct ApplicationConfig -{ - bool uses_mono_llvm; - bool uses_mono_aot; - bool uses_assembly_preload; - bool is_a_bundled_app; - bool broken_exception_transitions; - uint8_t bound_exception_type; - uint32_t package_naming_policy; - uint32_t environment_variable_count; - uint32_t system_property_count; - const char *android_package_name; -}; - -MONO_API TypeMapHeader jm_typemap_header; -MONO_API uint8_t jm_typemap[]; - -MONO_API TypeMapHeader mj_typemap_header; -MONO_API uint8_t mj_typemap[]; - -MONO_API ApplicationConfig application_config; -MONO_API const char* app_environment_variables[]; -MONO_API const char* app_system_properties[]; - -MONO_API const char* mono_aot_mode_name; -#endif // __XAMARIN_ANDROID_TYPEMAP_H diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh new file mode 100644 index 00000000000..67bfd011d85 --- /dev/null +++ b/src/monodroid/jni/xamarin-app.hh @@ -0,0 +1,88 @@ +// Dear Emacs, this is a -*- C++ -*- header +#ifndef __XAMARIN_ANDROID_TYPEMAP_H +#define __XAMARIN_ANDROID_TYPEMAP_H + +#include + +#include + +#include "monodroid.h" + +static constexpr uint32_t MODULE_MAGIC = 0x4D544158; // 'XATM', little-endian +static constexpr uint32_t MODULE_INDEX_MAGIC = 0x49544158; // 'XATI', little-endian +static constexpr uint8_t MODULE_FORMAT_VERSION = 1; // Keep in sync with the value in src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs + +struct BinaryTypeMapHeader +{ + uint32_t magic; + uint32_t version; + uint8_t module_uuid[16]; + uint32_t entry_count; + uint32_t duplicate_count; + uint32_t java_name_width; + uint32_t assembly_name_length; +}; + +struct TypeMapIndexHeader +{ + uint32_t magic; + uint32_t version; + uint32_t entry_count; + uint32_t module_file_name_width; +}; + +struct TypeMapModuleEntry +{ + int32_t type_token_id; + uint32_t java_map_index; +} __attribute__((packed)); + +struct TypeMapModule +{ + uint8_t module_uuid[16]; + uint32_t entry_count; + uint32_t duplicate_count; + TypeMapModuleEntry *map; + TypeMapModuleEntry *duplicate_map; + char *assembly_name; + MonoImage *image; +#if defined (DEBUG) || !defined (ANDROID) + uint32_t java_name_width; + uint8_t *java_map; +#endif +}; + +struct TypeMapJava +{ + uint32_t module_index; + int32_t type_token_id; + uint8_t java_name[]; +}; + +struct ApplicationConfig +{ + bool uses_mono_llvm; + bool uses_mono_aot; + bool uses_assembly_preload; + bool is_a_bundled_app; + bool broken_exception_transitions; + bool instant_run_enabled; + uint8_t bound_exception_type; + uint32_t package_naming_policy; + uint32_t environment_variable_count; + uint32_t system_property_count; + const char *android_package_name; +}; + +MONO_API const uint32_t map_module_count; +MONO_API const uint32_t java_type_count; +MONO_API const uint32_t java_name_width; +MONO_API const TypeMapModule map_modules[]; +MONO_API const TypeMapJava map_java[]; + +MONO_API ApplicationConfig application_config; +MONO_API const char* app_environment_variables[]; +MONO_API const char* app_system_properties[]; + +MONO_API const char* mono_aot_mode_name; +#endif // __XAMARIN_ANDROID_TYPEMAP_H