diff --git a/Directory.Build.props b/Directory.Build.props index 3f855aae763..d01d6648f8e 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -48,7 +48,7 @@ 5.4.0 1.1.11 6.12.0.148 - 6.0.0 + 8.0.0 6.0.0 2.13.1 2.14.1 diff --git a/Xamarin.Android.sln b/Xamarin.Android.sln index 50918d1e4e3..c85d197a5aa 100644 --- a/Xamarin.Android.sln +++ b/Xamarin.Android.sln @@ -109,7 +109,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "decompress-assemblies", "to EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "tmt", "tools\tmt\tmt.csproj", "{1A273ED2-AE84-48E9-9C23-E978C2D0CB34}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "assembly-store-reader", "tools\assembly-store-reader\assembly-store-reader.csproj", "{DA50FC92-7FE7-48B5-BDB6-CDA57B37BB51}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "assembly-store-reader", "tools\assembly-store-reader-mk2\assembly-store-reader.csproj", "{DA50FC92-7FE7-48B5-BDB6-CDA57B37BB51}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Java.Interop.Tools.JavaTypeSystem", "external\Java.Interop\src\Java.Interop.Tools.JavaTypeSystem\Java.Interop.Tools.JavaTypeSystem.csproj", "{4EFCED6E-9A6B-453A-94E4-CE4B736EC684}" EndProject diff --git a/build-tools/scripts/TestApks.targets b/build-tools/scripts/TestApks.targets index 95ba391d170..1110f32453c 100644 --- a/build-tools/scripts/TestApks.targets +++ b/build-tools/scripts/TestApks.targets @@ -111,7 +111,14 @@ Timeout="60000" /> + <_ResolvedJavaLibraries Include="@(ResolvedFileToPublish)" Condition=" '%(ResolvedFileToPublish.Extension)' == '.jar' " /> + + diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index c99ce0db5c5..45778d60204 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -23,6 +23,9 @@ namespace Xamarin.Android.Tasks { public class BuildApk : AndroidTask { + const string ArchiveAssembliesPath = "lib"; + const string ArchiveLibPath = "lib"; + public override string TaskPrefix => "BLD"; public string AndroidNdkDirectory { get; set; } @@ -33,6 +36,7 @@ public class BuildApk : AndroidTask [Required] public string ApkOutputPath { get; set; } + [Required] public string AppSharedLibrariesDir { get; set; } [Required] @@ -118,12 +122,11 @@ bool _Debug { SequencePointsMode sequencePointsMode = SequencePointsMode.None; public ITaskItem[] LibraryProjectJars { get; set; } - string [] uncompressedFileExtensions; + HashSet uncompressedFileExtensions; + // Do not use trailing / in the path protected virtual string RootPath => ""; - protected virtual string AssembliesPath => RootPath + "assemblies/"; - protected virtual string DalvikPath => ""; protected virtual CompressionMethod UncompressedMethod => CompressionMethod.Store; @@ -136,7 +139,7 @@ protected virtual void FixupArchive (ZipArchiveEx zip) { } List includePatterns = new List (); - void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOutputPath, bool debug, bool compress, IDictionary compressedAssembliesInfo, string assemblyStoreApkName) + void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOutputPath, bool debug, bool compress, IDictionary> compressedAssembliesInfo, string assemblyStoreApkName) { ArchiveFileList files = new ArchiveFileList (); bool refresh = true; @@ -207,6 +210,7 @@ void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOut apk.Flush (); } + AddRuntimeConfigBlob (apk); AddRuntimeLibraries (apk, supportedAbis); apk.Flush(); AddNativeLibraries (files, supportedAbis); @@ -218,10 +222,6 @@ void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOut } } - if (!String.IsNullOrEmpty (RuntimeConfigBinFilePath) && File.Exists (RuntimeConfigBinFilePath)) { - AddFileToArchiveIfNewer (apk, RuntimeConfigBinFilePath, $"{AssembliesPath}rc.bin", compressionMethod: UncompressedMethod); - } - foreach (var file in files) { var item = Path.Combine (file.archivePath.Replace (Path.DirectorySeparatorChar, '/')); existingEntries.Remove (item); @@ -318,7 +318,18 @@ public override bool RunTask () Aot.TryGetSequencePointsMode (AndroidSequencePointsMode, out sequencePointsMode); var outputFiles = new List (); - uncompressedFileExtensions = UncompressedFileExtensions?.Split (new char [] { ';', ',' }, StringSplitOptions.RemoveEmptyEntries) ?? Array.Empty (); + uncompressedFileExtensions = new HashSet (StringComparer.OrdinalIgnoreCase); + foreach (string? e in UncompressedFileExtensions?.Split (new char [] { ';', ',' }, StringSplitOptions.RemoveEmptyEntries) ?? Array.Empty ()) { + string? ext = e?.Trim (); + if (String.IsNullOrEmpty (ext)) { + continue; + } + + if (ext[0] != '.') { + ext = $".{ext}"; + } + uncompressedFileExtensions.Add (ext); + } existingEntries.Clear (); @@ -331,12 +342,12 @@ public override bool RunTask () bool debug = _Debug; bool compress = !debug && EnableCompression; - IDictionary compressedAssembliesInfo = null; + IDictionary> compressedAssembliesInfo = null; if (compress) { string key = CompressedAssemblyInfo.GetKey (ProjectFullPath); Log.LogDebugMessage ($"Retrieving assembly compression info with key '{key}'"); - compressedAssembliesInfo = BuildEngine4.UnregisterTaskObjectAssemblyLocal> (key, RegisteredTaskObjectLifetime.Build); + compressedAssembliesInfo = BuildEngine4.UnregisterTaskObjectAssemblyLocal>> (key, RegisteredTaskObjectLifetime.Build); if (compressedAssembliesInfo == null) throw new InvalidOperationException ($"Assembly compression info not found for key '{key}'. Compression will not be performed."); } @@ -380,34 +391,36 @@ static Regex FileGlobToRegEx (string fileGlob, RegexOptions options) return new Regex (sb.ToString (), options); } - void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary compressedAssembliesInfo, string assemblyStoreApkName) + void AddRuntimeConfigBlob (ZipArchiveEx apk) + { + // We will place rc.bin in the `lib` directory next to the blob, to make startup slightly faster, as we will find the config file right after we encounter + // our assembly store. Not only that, but also we'll be able to skip scanning the `base.apk` archive when split configs are enabled (which they are in 99% + // of cases these days, since AAB enforces that split). `base.apk` contains only ABI-agnostic file, while one of the split config files contains only + // ABI-specific data+code. + if (!String.IsNullOrEmpty (RuntimeConfigBinFilePath) && File.Exists (RuntimeConfigBinFilePath)) { + foreach (string abi in SupportedAbis) { + // Prefix it with `a` because bundletool sorts entries alphabetically, and this will place it right next to `assemblies.*.blob.so`, which is what we + // like since we can finish scanning the zip central directory earlier at startup. + string inArchivePath = MakeArchiveLibPath (abi, "libarc.bin.so"); + AddFileToArchiveIfNewer (apk, RuntimeConfigBinFilePath, inArchivePath, compressionMethod: GetCompressionMethod (inArchivePath)); + } + } + } + + void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary> compressedAssembliesInfo, string assemblyStoreApkName) { string sourcePath; AssemblyCompression.AssemblyData compressedAssembly = null; string compressedOutputDir = Path.GetFullPath (Path.Combine (Path.GetDirectoryName (ApkOutputPath), "..", "lz4")); - AssemblyStoreGenerator storeGenerator; + AssemblyStoreGenerator? storeGenerator; if (UseAssemblyStore) { - storeGenerator = new AssemblyStoreGenerator (AssembliesPath, Log); + storeGenerator = new AssemblyStoreGenerator (Log); } else { storeGenerator = null; } - AssemblyStoreAssemblyInfo storeAssembly = null; - - // - // NOTE - // - // The very first store (ID 0) **must** contain an index of all the assemblies included in the application, even if they - // are included in other APKs than the base one. The ID 0 store **must** be placed in the base assembly - // - - // Currently, all the assembly stores end up in the "base" apk (the APK name is the key in the dictionary below) but the code is ready for the time when we - // partition assemblies into "feature" APKs - const string DefaultBaseApkName = "base"; - if (String.IsNullOrEmpty (assemblyStoreApkName)) { - assemblyStoreApkName = DefaultBaseApkName; - } + AssemblyStoreAssemblyInfo? storeAssemblyInfo = null; // Add user assemblies AddAssembliesFromCollection (ResolvedUserAssemblies); @@ -419,41 +432,53 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary> assemblyStorePaths = storeGenerator.Generate (Path.GetDirectoryName (ApkOutputPath)); - if (assemblyStorePaths == null) { + Dictionary assemblyStorePaths = storeGenerator.Generate (AppSharedLibrariesDir); + + if (assemblyStorePaths.Count == 0) { throw new InvalidOperationException ("Assembly store generator did not generate any stores"); } - if (!assemblyStorePaths.TryGetValue (assemblyStoreApkName, out List baseAssemblyStores) || baseAssemblyStores == null || baseAssemblyStores.Count == 0) { - throw new InvalidOperationException ("Assembly store generator didn't generate the required base stores"); + if (assemblyStorePaths.Count != SupportedAbis.Length) { + throw new InvalidOperationException ("Internal error: assembly store did not generate store for each supported ABI"); } - string assemblyStorePrefix = $"{assemblyStoreApkName}_"; - foreach (string assemblyStorePath in baseAssemblyStores) { - string inArchiveName = Path.GetFileName (assemblyStorePath); - - if (inArchiveName.StartsWith (assemblyStorePrefix, StringComparison.Ordinal)) { - inArchiveName = inArchiveName.Substring (assemblyStorePrefix.Length); - } - - CompressionMethod compressionMethod; - if (inArchiveName.EndsWith (".manifest", StringComparison.Ordinal)) { - compressionMethod = CompressionMethod.Default; - } else { - compressionMethod = UncompressedMethod; - } - - AddFileToArchiveIfNewer (apk, assemblyStorePath, AssembliesPath + inArchiveName, compressionMethod); + string inArchivePath; + foreach (var kvp in assemblyStorePaths) { + string abi = MonoAndroidHelper.ArchToAbi (kvp.Key); + inArchivePath = MakeArchiveLibPath (abi, "lib" + Path.GetFileName (kvp.Value)); + AddFileToArchiveIfNewer (apk, kvp.Value, inArchivePath, GetCompressionMethod (inArchivePath)); } void AddAssembliesFromCollection (ITaskItem[] assemblies) { - foreach (ITaskItem assembly in assemblies) { - if (bool.TryParse (assembly.GetMetadata ("AndroidSkipAddToPackage"), out bool value) && value) { - Log.LogDebugMessage ($"Skipping {assembly.ItemSpec} due to 'AndroidSkipAddToPackage' == 'true' "); - continue; + Dictionary> perArchAssemblies = MonoAndroidHelper.GetPerArchAssemblies ( + assemblies, + SupportedAbis, + validate: true, + shouldSkip: (ITaskItem asm) => { + if (bool.TryParse (asm.GetMetadata ("AndroidSkipAddToPackage"), out bool value) && value) { + Log.LogDebugMessage ($"Skipping {asm.ItemSpec} due to 'AndroidSkipAddToPackage' == 'true' "); + return true; + } + + return false; } + ); + + foreach (var kvp in perArchAssemblies) { + Log.LogDebugMessage ($"Adding assemblies for architecture '{kvp.Key}'"); + DoAddAssembliesFromArchCollection (kvp.Value); + } + } + void DoAddAssembliesFromArchCollection (Dictionary assemblies) + { + // In the "all assemblies are per-RID" world, assemblies, pdb and config are disguised as shared libraries (that is, + // their names end with the .so extension) so that Android allows us to put them in the `lib/{ARCH}` directory. + // For this reason, they have to be treated just like other .so files, as far as compression rules are concerned. + // Thus, we no longer just store them in the apk but we call the `GetCompressionMethod` method to find out whether + // or not we're supposed to compress .so files. + foreach (ITaskItem assembly in assemblies.Values) { if (MonoAndroidHelper.IsReferenceAssembly (assembly.ItemSpec)) { Log.LogCodedWarning ("XA0107", assembly.ItemSpec, 0, Properties.Resources.XA0107, assembly.ItemSpec); } @@ -461,25 +486,27 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies) sourcePath = CompressAssembly (assembly); // Add assembly - var assemblyPath = GetAssemblyPath (assembly, frameworkAssembly: false); + (string assemblyPath, string assemblyDirectory) = GetInArchiveAssemblyPath (assembly); if (UseAssemblyStore) { - storeAssembly = new AssemblyStoreAssemblyInfo (sourcePath, assemblyPath, assembly.GetMetadata ("Abi")); + storeAssemblyInfo = new AssemblyStoreAssemblyInfo (sourcePath, assembly); } else { - AddFileToArchiveIfNewer (apk, sourcePath, assemblyPath + Path.GetFileName (assembly.ItemSpec), compressionMethod: UncompressedMethod); + AddFileToArchiveIfNewer (apk, sourcePath, assemblyPath, compressionMethod: GetCompressionMethod (assemblyPath)); } // Try to add config if exists var config = Path.ChangeExtension (assembly.ItemSpec, "dll.config"); if (UseAssemblyStore) { - storeAssembly.SetConfigPath (config); + if (File.Exists (config)) { + storeAssemblyInfo.ConfigFile = new FileInfo (config); + } } else { - AddAssemblyConfigEntry (apk, assemblyPath, config); + AddAssemblyConfigEntry (apk, assemblyDirectory, config); } // Try to add symbols if Debug if (debug) { var symbols = Path.ChangeExtension (assembly.ItemSpec, "pdb"); - string symbolsPath = null; + string? symbolsPath = null; if (File.Exists (symbols)) { symbolsPath = symbols; @@ -487,15 +514,21 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies) if (!String.IsNullOrEmpty (symbolsPath)) { if (UseAssemblyStore) { - storeAssembly.SetDebugInfoPath (symbolsPath); + storeAssemblyInfo.SymbolsFile = new FileInfo (symbolsPath); } else { - AddFileToArchiveIfNewer (apk, symbolsPath, assemblyPath + Path.GetFileName (symbols), compressionMethod: UncompressedMethod); + string archiveSymbolsPath = assemblyDirectory + MonoAndroidHelper.MakeDiscreteAssembliesEntryName (Path.GetFileName (symbols)); + AddFileToArchiveIfNewer ( + apk, + symbolsPath, + archiveSymbolsPath, + compressionMethod: GetCompressionMethod (archiveSymbolsPath) + ); } } } if (UseAssemblyStore) { - storeGenerator.Add (assemblyStoreApkName, storeAssembly); + storeGenerator.Add (storeAssemblyInfo); } } } @@ -519,38 +552,44 @@ string CompressAssembly (ITaskItem assembly) return assembly.ItemSpec; } - var key = CompressedAssemblyInfo.GetDictionaryKey (assembly); - if (compressedAssembliesInfo.TryGetValue (key, out CompressedAssemblyInfo info) && info != null) { - EnsureCompressedAssemblyData (assembly.ItemSpec, info.DescriptorIndex); - string assemblyOutputDir; - string subDirectory = assembly.GetMetadata ("DestinationSubDirectory"); - if (!String.IsNullOrEmpty (subDirectory)) - assemblyOutputDir = Path.Combine (compressedOutputDir, subDirectory); - else - assemblyOutputDir = compressedOutputDir; - AssemblyCompression.CompressionResult result = AssemblyCompression.Compress (compressedAssembly, assemblyOutputDir); - if (result != AssemblyCompression.CompressionResult.Success) { - switch (result) { - case AssemblyCompression.CompressionResult.EncodingFailed: - Log.LogMessage ($"Failed to compress {assembly.ItemSpec}"); - break; - - case AssemblyCompression.CompressionResult.InputTooBig: - Log.LogMessage ($"Input assembly {assembly.ItemSpec} exceeds maximum input size"); - break; - - default: - Log.LogMessage ($"Unknown error compressing {assembly.ItemSpec}"); - break; - } - return assembly.ItemSpec; - } - return compressedAssembly.DestinationPath; - } else { + string key = CompressedAssemblyInfo.GetDictionaryKey (assembly); + AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (assembly); + if (!compressedAssembliesInfo.TryGetValue (arch, out Dictionary assembliesInfo)) { + throw new InvalidOperationException ($"Internal error: compression assembly info for architecture {arch} not available"); + } + + if (!assembliesInfo.TryGetValue (key, out CompressedAssemblyInfo info) || info == null) { Log.LogDebugMessage ($"Assembly missing from {nameof (CompressedAssemblyInfo)}: {key}"); + return assembly.ItemSpec; } - return assembly.ItemSpec; + EnsureCompressedAssemblyData (assembly.ItemSpec, info.DescriptorIndex); + string assemblyOutputDir; + string subDirectory = assembly.GetMetadata ("DestinationSubDirectory"); + string abi = MonoAndroidHelper.GetAssemblyAbi (assembly); + if (!String.IsNullOrEmpty (subDirectory)) { + assemblyOutputDir = Path.Combine (compressedOutputDir, abi, subDirectory); + } else { + assemblyOutputDir = Path.Combine (compressedOutputDir, abi); + } + AssemblyCompression.CompressionResult result = AssemblyCompression.Compress (compressedAssembly, assemblyOutputDir); + if (result != AssemblyCompression.CompressionResult.Success) { + switch (result) { + case AssemblyCompression.CompressionResult.EncodingFailed: + Log.LogMessage ($"Failed to compress {assembly.ItemSpec}"); + break; + + case AssemblyCompression.CompressionResult.InputTooBig: + Log.LogMessage ($"Input assembly {assembly.ItemSpec} exceeds maximum input size"); + break; + + default: + Log.LogMessage ($"Unknown error compressing {assembly.ItemSpec}"); + break; + } + return assembly.ItemSpec; + } + return compressedAssembly.DestinationPath; } } @@ -568,13 +607,14 @@ bool AddFileToArchiveIfNewer (ZipArchiveEx apk, string file, string inArchivePat void AddAssemblyConfigEntry (ZipArchiveEx apk, string assemblyPath, string configFile) { - string inArchivePath = assemblyPath + Path.GetFileName (configFile); + string inArchivePath = MonoAndroidHelper.MakeDiscreteAssembliesEntryName (assemblyPath + Path.GetFileName (configFile)); existingEntries.Remove (inArchivePath); - if (!File.Exists (configFile)) + if (!File.Exists (configFile)) { return; + } - CompressionMethod compressionMethod = UncompressedMethod; + CompressionMethod compressionMethod = GetCompressionMethod (inArchivePath); if (apk.SkipExistingFile (configFile, inArchivePath, compressionMethod)) { Log.LogDebugMessage ($"Skipping {configFile} as the archive file is up to date."); return; @@ -593,19 +633,48 @@ void AddAssemblyConfigEntry (ZipArchiveEx apk, string assemblyPath, string confi /// /// Returns the in-archive path for an assembly /// - string GetAssemblyPath (ITaskItem assembly, bool frameworkAssembly) + (string assemblyFilePath, string assemblyDirectoryPath) GetInArchiveAssemblyPath (ITaskItem assembly) { - var assembliesPath = AssembliesPath; - var subDirectory = assembly.GetMetadata ("DestinationSubDirectory"); - if (!string.IsNullOrEmpty (subDirectory)) { - assembliesPath += subDirectory.Replace ('\\', '/'); - if (!assembliesPath.EndsWith ("/", StringComparison.Ordinal)) { - assembliesPath += "/"; + var parts = new List (); + + // The PrepareSatelliteAssemblies task takes care of properly setting `DestinationSubDirectory`, so we can just use it here. + string? subDirectory = assembly.GetMetadata ("DestinationSubDirectory")?.Replace ('\\', '/'); + if (string.IsNullOrEmpty (subDirectory)) { + throw new InvalidOperationException ($"Internal error: assembly '{assembly}' lacks the required `DestinationSubDirectory` metadata"); + } + + string assemblyName = Path.GetFileName (assembly.ItemSpec); + if (UseAssemblyStore) { + parts.Add (subDirectory); + parts.Add (assemblyName); + } else { + // For discrete assembly entries we need to treat assemblies specially. + // All of the assemblies have their names mangled so that the possibility to clash with "real" shared + // library names is minimized. All of the assembly entries will start with a special character: + // + // `_` - for regular assemblies (e.g. `_Mono.Android.dll.so`) + // `-` - for satellite assemblies (e.g. `-es-Mono.Android.dll.so`) + // + // Second of all, we need to treat satellite assemblies with even more care. + // If we encounter one of them, we will return the culture as part of the path transformed + // so that it forms a `-culture-` assembly file name prefix, not a `culture/` subdirectory. + // This is necessary because Android doesn't allow subdirectories in `lib/{ABI}/` + // + string[] subdirParts = subDirectory.TrimEnd ('/').Split ('/'); + if (subdirParts.Length == 1) { + // Not a satellite assembly + parts.Add (subDirectory); + parts.Add (MonoAndroidHelper.MakeDiscreteAssembliesEntryName (assemblyName)); + } else if (subdirParts.Length == 2) { + parts.Add (subdirParts[0]); + parts.Add (MonoAndroidHelper.MakeDiscreteAssembliesEntryName (assemblyName, subdirParts[1])); + } else { + throw new InvalidOperationException ($"Internal error: '{assembly}' `DestinationSubDirectory` metadata has too many components ({parts.Count} instead of 1 or 2)"); } - } else if (!frameworkAssembly && SatelliteAssembly.TryGetSatelliteCultureAndFileName (assembly.ItemSpec, out var culture, out _)) { - assembliesPath += culture + "/"; } - return assembliesPath; + + string assemblyFilePath = MonoAndroidHelper.MakeZipArchivePath (ArchiveAssembliesPath, parts); + return (assemblyFilePath, Path.GetDirectoryName (assemblyFilePath) + "/"); } sealed class LibInfo @@ -618,14 +687,12 @@ sealed class LibInfo CompressionMethod GetCompressionMethod (string fileName) { - if (uncompressedFileExtensions.Any (x => string.Compare (x.StartsWith (".", StringComparison.OrdinalIgnoreCase) ? x : $".{x}", Path.GetExtension (fileName), StringComparison.OrdinalIgnoreCase) == 0)) - return UncompressedMethod; - return CompressionMethod.Default; + return uncompressedFileExtensions.Contains (Path.GetExtension (fileName)) ? UncompressedMethod : CompressionMethod.Default; } void AddNativeLibraryToArchive (ZipArchiveEx apk, string abi, string filesystemPath, string inArchiveFileName) { - string archivePath = $"lib/{abi}/{inArchiveFileName}"; + string archivePath = MakeArchiveLibPath (abi, inArchiveFileName); existingEntries.Remove (archivePath); CompressionMethod compressionMethod = GetCompressionMethod (archivePath); if (apk.SkipExistingFile (filesystemPath, archivePath, compressionMethod)) { @@ -809,7 +876,7 @@ private void AddAdditionalNativeLibraries (ArchiveFileList files, string [] supp void AddNativeLibrary (ArchiveFileList files, string path, string abi, string archiveFileName) { string fileName = string.IsNullOrEmpty (archiveFileName) ? Path.GetFileName (path) : archiveFileName; - var item = (filePath: path, archivePath: $"lib/{abi}/{fileName}"); + var item = (filePath: path, archivePath: MakeArchiveLibPath (abi, fileName)); if (files.Any (x => x.archivePath == item.archivePath)) { Log.LogCodedWarning ("XA4301", path, 0, Properties.Resources.XA4301, item.archivePath); return; @@ -833,5 +900,7 @@ void LogSanitizerError (string message) { Log.LogError (message); } + + static string MakeArchiveLibPath (string abi, string fileName) => MonoAndroidHelper.MakeZipArchivePath (ArchiveLibPath, abi, fileName); } } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs index 441299c527d..81a11497da4 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs @@ -4,6 +4,8 @@ using Microsoft.Build.Framework; using Microsoft.Android.Build.Tasks; +using Xamarin.Android.Tools; + namespace Xamarin.Android.Tasks { public class GenerateCompressedAssembliesNativeSourceFiles : AndroidTask @@ -41,38 +43,54 @@ void GenerateCompressedAssemblySources () return; } - var assemblies = new SortedDictionary (StringComparer.Ordinal); - foreach (ITaskItem assembly in ResolvedAssemblies) { - if (bool.TryParse (assembly.GetMetadata ("AndroidSkipAddToPackage"), out bool value) && value) { - continue; - } + Dictionary> perArchAssemblies = MonoAndroidHelper.GetPerArchAssemblies ( + ResolvedAssemblies, + SupportedAbis, + validate: true, + shouldSkip: (ITaskItem asm) => bool.TryParse (asm.GetMetadata ("AndroidSkipAddToPackage"), out bool value) && value + ); + var archAssemblies = new Dictionary> (); + var counters = new Dictionary (); + + foreach (var kvpPerArch in perArchAssemblies) { + AndroidTargetArch arch = kvpPerArch.Key; + Dictionary resolvedArchAssemblies = kvpPerArch.Value; + + foreach (var kvp in resolvedArchAssemblies) { + ITaskItem assembly = kvp.Value; + + if (!archAssemblies.TryGetValue (arch, out Dictionary assemblies)) { + assemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); + archAssemblies.Add (arch, assemblies); + } - var assemblyKey = CompressedAssemblyInfo.GetDictionaryKey (assembly); - if (assemblies.ContainsKey (assemblyKey)) { - Log.LogDebugMessage ($"Skipping duplicate assembly: {assembly.ItemSpec}"); - continue; - } + var assemblyKey = CompressedAssemblyInfo.GetDictionaryKey (assembly); + if (assemblies.ContainsKey (assemblyKey)) { + Log.LogDebugMessage ($"Skipping duplicate assembly: {assembly.ItemSpec} (arch {MonoAndroidHelper.GetAssemblyAbi(assembly)})"); + continue; + } - var fi = new FileInfo (assembly.ItemSpec); - if (!fi.Exists) { - Log.LogError ($"Assembly {assembly.ItemSpec} does not exist"); - continue; - } + var fi = new FileInfo (assembly.ItemSpec); + if (!fi.Exists) { + Log.LogError ($"Assembly {assembly.ItemSpec} does not exist"); + continue; + } - assemblies.Add (assemblyKey, new CompressedAssemblyInfo (checked((uint)fi.Length))); - } - uint index = 0; - foreach (var kvp in assemblies) { - kvp.Value.DescriptorIndex = index++; + if (!counters.TryGetValue (arch, out uint counter)) { + counter = 0; + } + assemblies.Add (assemblyKey, new CompressedAssemblyInfo (checked((uint)fi.Length), counter++, arch, Path.GetFileNameWithoutExtension (assembly.ItemSpec))); + counters[arch] = counter; + } } string key = CompressedAssemblyInfo.GetKey (ProjectFullPath); Log.LogDebugMessage ($"Storing compression assemblies info with key '{key}'"); - BuildEngine4.RegisterTaskObjectAssemblyLocal (key, assemblies, RegisteredTaskObjectLifetime.Build); - Generate (assemblies); + BuildEngine4.RegisterTaskObjectAssemblyLocal (key, archAssemblies, RegisteredTaskObjectLifetime.Build); + Generate (archAssemblies); - void Generate (IDictionary dict) + void Generate (Dictionary> dict) { var composer = new CompressedAssembliesNativeAssemblyGenerator (Log, dict); LLVMIR.LlvmIrModule compressedAssemblies = composer.Construct (); diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 453885d1758..8bd49bfcd99 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -2,16 +2,11 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; -using System.IO.MemoryMappedFiles; using System.Linq; -using System.Reflection; -using System.Text; using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; using Mono.Cecil; - +using Microsoft.Build.Utilities; using Java.Interop.Tools.Cecil; using Java.Interop.Tools.Diagnostics; @@ -28,7 +23,7 @@ namespace Xamarin.Android.Tasks public class GenerateJavaStubs : AndroidTask { - public const string MarshalMethodsRegisterTaskKey = ".:!MarshalMethods!:."; + public const string NativeCodeGenStateRegisterTaskKey = ".:!MarshalMethods!:."; public override string TaskPrefix => "GJS"; @@ -95,7 +90,7 @@ public class GenerateJavaStubs : AndroidTask public ITaskItem[] Environments { get; set; } [Output] - public string [] GeneratedBinaryTypeMaps { get; set; } + public ITaskItem[] GeneratedBinaryTypeMaps { get; set; } internal const string AndroidSkipJavaStubGeneration = "AndroidSkipJavaStubGeneration"; @@ -103,11 +98,7 @@ public override bool RunTask () { try { bool useMarshalMethods = !Debug && EnableMarshalMethods; - // We're going to do 3 steps here instead of separate tasks so - // we can share the list of JLO TypeDefinitions between them - using (XAAssemblyResolver res = MakeResolver (useMarshalMethods)) { - Run (res, useMarshalMethods); - } + Run (useMarshalMethods); } catch (XamarinAndroidException e) { Log.LogCodedError (string.Format ("XA{0:0000}", e.Code), e.MessageWithoutCode); if (MonoAndroidHelper.LogInternalExceptions) @@ -124,210 +115,207 @@ public override bool RunTask () return !Log.HasLoggedErrors; } - XAAssemblyResolver MakeResolver (bool useMarshalMethods) + XAAssemblyResolver MakeResolver (bool useMarshalMethods, AndroidTargetArch targetArch, Dictionary assemblies) { - var readerParams = new ReaderParameters(); + var readerParams = new ReaderParameters (); if (useMarshalMethods) { readerParams.ReadWrite = true; readerParams.InMemory = true; } - var res = new XAAssemblyResolver (Log, loadDebugSymbols: true, loadReaderParameters: readerParams); - foreach (var dir in FrameworkDirectories) { - if (Directory.Exists (dir.ItemSpec)) { - res.FrameworkSearchDirectories.Add (dir.ItemSpec); + var res = new XAAssemblyResolver (targetArch, Log, loadDebugSymbols: true, loadReaderParameters: readerParams); + var uniqueDirs = new HashSet (StringComparer.OrdinalIgnoreCase); + + Log.LogDebugMessage ($"Adding search directories to new architecture {targetArch} resolver:"); + foreach (var kvp in assemblies) { + string assemblyDir = Path.GetDirectoryName (kvp.Value.ItemSpec); + if (uniqueDirs.Contains (assemblyDir)) { + continue; } + + uniqueDirs.Add (assemblyDir); + res.SearchDirectories.Add (assemblyDir); + Log.LogDebugMessage ($" {assemblyDir}"); } return res; } - void Run (XAAssemblyResolver res, bool useMarshalMethods) + void Run (bool useMarshalMethods) { PackageNamingPolicy pnp; JavaNativeTypeManager.PackageNamingPolicy = Enum.TryParse (PackageNamingPolicy, out pnp) ? pnp : PackageNamingPolicyEnum.LowercaseCrc64; - Dictionary>? abiSpecificAssembliesByPath = null; - if (useMarshalMethods) { - abiSpecificAssembliesByPath = new Dictionary> (StringComparer.Ordinal); + // We will process each architecture completely separately as both type maps and marshal methods are strictly per-architecture and + // the assemblies should be processed strictly per architecture. Generation of JCWs, and the manifest are ABI-agnostic. + // We will generate them only for the first architecture, whichever it is. + Dictionary> allAssembliesPerArch = MonoAndroidHelper.GetPerArchAssemblies (ResolvedAssemblies, SupportedAbis, validate: true); + + // Should "never" happen... + if (allAssembliesPerArch.Count != SupportedAbis.Length) { + // ...but it happens at least in our `BuildAMassiveApp` test, where `SupportedAbis` mentions only the `x86` and `armeabi-v7a` ABIs, but `ResolvedAssemblies` contains + // entries for all the ABIs we support, so let's be flexible and ignore the extra architectures but still error out if there are less architectures than supported ABIs. + if (allAssembliesPerArch.Count < SupportedAbis.Length) { + throw new InvalidOperationException ($"Internal error: number of architectures ({allAssembliesPerArch.Count}) must equal the number of target ABIs ({SupportedAbis.Length})"); + } } - // Put every assembly we'll need in the resolver - bool hasExportReference = false; - bool haveMonoAndroid = false; - var allTypemapAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); - var userAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); - - foreach (var assembly in ResolvedAssemblies) { - bool value; - if (bool.TryParse (assembly.GetMetadata (AndroidSkipJavaStubGeneration), out value) && value) { - Log.LogDebugMessage ($"Skipping Java Stub Generation for {assembly.ItemSpec}"); - continue; + // ...or this... + foreach (string abi in SupportedAbis) { + AndroidTargetArch arch = MonoAndroidHelper.AbiToTargetArch (abi); + if (!allAssembliesPerArch.ContainsKey (arch)) { + throw new InvalidOperationException ($"Internal error: no assemblies for architecture '{arch}', which corresponds to target abi '{abi}'"); } + } - bool addAssembly = false; - string fileName = Path.GetFileName (assembly.ItemSpec); - if (!hasExportReference && String.Compare ("Mono.Android.Export.dll", fileName, StringComparison.OrdinalIgnoreCase) == 0) { - hasExportReference = true; - addAssembly = true; - } else if (!haveMonoAndroid && String.Compare ("Mono.Android.dll", fileName, StringComparison.OrdinalIgnoreCase) == 0) { - haveMonoAndroid = true; - addAssembly = true; - } else if (MonoAndroidHelper.FrameworkAssembliesToTreatAsUserAssemblies.Contains (fileName)) { - if (!bool.TryParse (assembly.GetMetadata (AndroidSkipJavaStubGeneration), out value) || !value) { - string name = Path.GetFileNameWithoutExtension (fileName); - if (!userAssemblies.ContainsKey (name)) - userAssemblies.Add (name, assembly.ItemSpec); - addAssembly = true; - } + // ...as well as this + Dictionary> userAssembliesPerArch = MonoAndroidHelper.GetPerArchAssemblies (ResolvedUserAssemblies, SupportedAbis, validate: true); + foreach (var kvp in userAssembliesPerArch) { + if (!allAssembliesPerArch.TryGetValue (kvp.Key, out Dictionary allAssemblies)) { + throw new InvalidOperationException ($"Internal error: found user assemblies for architecture '{kvp.Key}' which isn't found in ResolvedAssemblies"); } - if (addAssembly) { - MaybeAddAbiSpecifcAssembly (assembly, fileName); - if (!allTypemapAssemblies.ContainsKey (assembly.ItemSpec)) { - allTypemapAssemblies.Add (assembly.ItemSpec, assembly); + foreach (var asmKvp in kvp.Value) { + if (!allAssemblies.ContainsKey (asmKvp.Key)) { + throw new InvalidOperationException ($"Internal error: user assembly '{asmKvp.Value}' not found in ResolvedAssemblies"); } } - - res.Load (MonoAndroidHelper.GetTargetArch (assembly), assembly.ItemSpec); } - // However we only want to look for JLO types in user code for Java stub code generation - foreach (var asm in ResolvedUserAssemblies) { - if (bool.TryParse (asm.GetMetadata (AndroidSkipJavaStubGeneration), out bool value) && value) { - Log.LogDebugMessage ($"Skipping Java Stub Generation for {asm.ItemSpec}"); - continue; - } - res.Load (MonoAndroidHelper.GetTargetArch (asm), asm.ItemSpec); - MaybeAddAbiSpecifcAssembly (asm, Path.GetFileName (asm.ItemSpec)); - if (!allTypemapAssemblies.ContainsKey (asm.ItemSpec)) { - allTypemapAssemblies.Add (asm.ItemSpec, asm); - } + // Now that "never" never happened, we can proceed knowing that at least the assembly sets are the same for each architecture + var nativeCodeGenStates = new Dictionary (); + bool generateJavaCode = true; + NativeCodeGenState? templateCodeGenState = null; - string name = Path.GetFileNameWithoutExtension (asm.ItemSpec); - if (!userAssemblies.ContainsKey (name)) - userAssemblies.Add (name, asm.ItemSpec); - } + foreach (var kvp in allAssembliesPerArch) { + AndroidTargetArch arch = kvp.Key; + Dictionary archAssemblies = kvp.Value; + (bool success, NativeCodeGenState? state) = GenerateJavaSourcesAndMaybeClassifyMarshalMethods (arch, archAssemblies, MaybeGetArchAssemblies (userAssembliesPerArch, arch), useMarshalMethods, generateJavaCode); - // Step 1 - Find all the JLO types - var cache = new TypeDefinitionCache (); - var scanner = new XAJavaTypeScanner (Log, cache) { - ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, - }; - List allJavaTypes = scanner.GetJavaTypes (allTypemapAssemblies.Values, res); - var javaTypes = new List (); + if (!success) { + return; + } - foreach (JavaType jt in allJavaTypes) { - // Whem marshal methods are in use we do not want to skip non-user assemblies (such as Mono.Android) - we need to generate JCWs for them during - // application build, unlike in Debug configuration or when marshal methods are disabled, in which case we use JCWs generated during Xamarin.Android - // build and stored in a jar file. - if ((!useMarshalMethods && !userAssemblies.ContainsKey (jt.Type.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (jt.Type, cache)) { - continue; + if (generateJavaCode) { + templateCodeGenState = state; + generateJavaCode = false; } - javaTypes.Add (jt); + + nativeCodeGenStates.Add (arch, state); } - MarshalMethodsClassifier classifier = null; - if (useMarshalMethods) { - classifier = new MarshalMethodsClassifier (cache, res, Log); + if (templateCodeGenState == null) { + throw new InvalidOperationException ($"Internal error: no native code generator state defined"); } + JCWGenerator.EnsureAllArchitecturesAreIdentical (Log, nativeCodeGenStates); - // Step 2 - Generate Java stub code - var success = CreateJavaSources (javaTypes, cache, classifier, useMarshalMethods); - if (!success) - return; + NativeCodeGenState.Template = templateCodeGenState; + BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (NativeCodeGenStateRegisterTaskKey), nativeCodeGenStates, RegisteredTaskObjectLifetime.Build); if (useMarshalMethods) { // We need to parse the environment files supplied by the user to see if they want to use broken exception transitions. This information is needed // in order to properly generate wrapper methods in the marshal methods assembly rewriter. // We don't care about those generated by us, since they won't contain the `XA_BROKEN_EXCEPTION_TRANSITIONS` variable we look for. var environmentParser = new EnvironmentFilesParser (); + bool brokenExceptionTransitionsEnabled = environmentParser.AreBrokenExceptionTransitionsEnabled (Environments); - Dictionary assemblyPaths = AddMethodsFromAbiSpecificAssemblies (classifier, res, abiSpecificAssembliesByPath); + foreach (var kvp in nativeCodeGenStates) { + NativeCodeGenState state = kvp.Value; + RewriteMarshalMethods (state, brokenExceptionTransitionsEnabled); + state.Classifier.AddSpecialCaseMethods (); - var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, assemblyPaths, Log); - rewriter.Rewrite (res, environmentParser.AreBrokenExceptionTransitionsEnabled (Environments)); - } - - // Step 3 - Generate type maps - // Type mappings need to use all the assemblies, always. - WriteTypeMappings (allJavaTypes, cache); - - // We need to save a map of .NET type -> ACW type for resource file fixups - var managed = new Dictionary (javaTypes.Count, StringComparer.Ordinal); - var java = new Dictionary (javaTypes.Count, StringComparer.Ordinal); - - var managedConflicts = new Dictionary> (0, StringComparer.Ordinal); - var javaConflicts = new Dictionary> (0, StringComparer.Ordinal); - - using (var acw_map = MemoryStreamPool.Shared.CreateStreamWriter ()) { - foreach (JavaType jt in javaTypes) { - TypeDefinition type = jt.Type; - string managedKey = type.FullName.Replace ('/', '.'); - string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'); - - acw_map.Write (type.GetPartialAssemblyQualifiedName (cache)); - acw_map.Write (';'); - acw_map.Write (javaKey); - acw_map.WriteLine (); - - TypeDefinition conflict; - bool hasConflict = false; - if (managed.TryGetValue (managedKey, out conflict)) { - if (!conflict.Module.Name.Equals (type.Module.Name)) { - if (!managedConflicts.TryGetValue (managedKey, out var list)) - managedConflicts.Add (managedKey, list = new List { conflict.GetPartialAssemblyName (cache) }); - list.Add (type.GetPartialAssemblyName (cache)); - } - hasConflict = true; - } - if (java.TryGetValue (javaKey, out conflict)) { - if (!conflict.Module.Name.Equals (type.Module.Name)) { - if (!javaConflicts.TryGetValue (javaKey, out var list)) - javaConflicts.Add (javaKey, list = new List { conflict.GetAssemblyQualifiedName (cache) }); - list.Add (type.GetAssemblyQualifiedName (cache)); - success = false; - } - hasConflict = true; + Log.LogDebugMessage ($"[{state.TargetArch}] Number of generated marshal methods: {state.Classifier.MarshalMethods.Count}"); + if (state.Classifier.RejectedMethodCount > 0) { + Log.LogWarning ($"[{state.TargetArch}] Number of methods in the project that will be registered dynamically: {state.Classifier.RejectedMethodCount}"); } - if (!hasConflict) { - managed.Add (managedKey, type); - java.Add (javaKey, type); - - acw_map.Write (managedKey); - acw_map.Write (';'); - acw_map.Write (javaKey); - acw_map.WriteLine (); - - acw_map.Write (JavaNativeTypeManager.ToCompatJniName (type, cache).Replace ('/', '.')); - acw_map.Write (';'); - acw_map.Write (javaKey); - acw_map.WriteLine (); + + if (state.Classifier.WrappedMethodCount > 0) { + // TODO: change to LogWarning once the generator can output code which requires no non-blittable wrappers + Log.LogDebugMessage ($"[{state.TargetArch}] Number of methods in the project that need marshal method wrappers: {state.Classifier.WrappedMethodCount}"); } } + } + + bool typemapsAreAbiAgnostic = Debug && !GenerateNativeAssembly; + bool first = true; + foreach (var kvp in nativeCodeGenStates) { + if (!first && typemapsAreAbiAgnostic) { + Log.LogDebugMessage ("Typemaps: it's a debug build and type maps are ABI-agnostic, not processing more ABIs"); + break; + } + + NativeCodeGenState state = kvp.Value; + first = false; + WriteTypeMappings (state); + } - acw_map.Flush (); - Files.CopyIfStreamChanged (acw_map.BaseStream, AcwMapFile); + var acwMapGen = new ACWMapGenerator (Log); + if (!acwMapGen.Generate (templateCodeGenState, AcwMapFile)) { + Log.LogDebugMessage ("ACW map generation failed"); } - foreach (var kvp in managedConflicts) { - Log.LogCodedWarning ("XA4214", Properties.Resources.XA4214, kvp.Key, string.Join (", ", kvp.Value)); - Log.LogCodedWarning ("XA4214", Properties.Resources.XA4214_Result, kvp.Key, kvp.Value [0]); + IList additionalProviders = MergeManifest (templateCodeGenState, MaybeGetArchAssemblies (userAssembliesPerArch, templateCodeGenState.TargetArch)); + GenerateAdditionalProviderSources (templateCodeGenState, additionalProviders); + + Dictionary MaybeGetArchAssemblies (Dictionary> dict, AndroidTargetArch arch) + { + if (!dict.TryGetValue (arch, out Dictionary archDict)) { + return new Dictionary (StringComparer.OrdinalIgnoreCase); + } + + return archDict; + } + } + + void GenerateAdditionalProviderSources (NativeCodeGenState codeGenState, IList additionalProviders) + { + // Create additional runtime provider java sources. + string providerTemplateFile = "MonoRuntimeProvider.Bundled.java"; + string providerTemplate = GetResource (providerTemplateFile); + + foreach (var provider in additionalProviders) { + var contents = providerTemplate.Replace ("MonoRuntimeProvider", provider); + var real_provider = Path.Combine (OutputDirectory, "src", "mono", provider + ".java"); + Files.CopyIfStringChanged (contents, real_provider); } - foreach (var kvp in javaConflicts) { - Log.LogCodedError ("XA4215", Properties.Resources.XA4215, kvp.Key); - foreach (var typeName in kvp.Value) - Log.LogCodedError ("XA4215", Properties.Resources.XA4215_Details, kvp.Key, typeName); + // Create additional application java sources. + StringWriter regCallsWriter = new StringWriter (); + regCallsWriter.WriteLine ("\t\t// Application and Instrumentation ACWs must be registered first."); + foreach (TypeDefinition type in codeGenState.JavaTypesForJCW) { + if (JavaNativeTypeManager.IsApplication (type, codeGenState.TypeCache) || JavaNativeTypeManager.IsInstrumentation (type, codeGenState.TypeCache)) { + if (codeGenState.Classifier != null && !codeGenState.Classifier.FoundDynamicallyRegisteredMethods (type)) { + continue; + } + + string javaKey = JavaNativeTypeManager.ToJniName (type, codeGenState.TypeCache).Replace ('/', '.'); + regCallsWriter.WriteLine ( + "\t\tmono.android.Runtime.register (\"{0}\", {1}.class, {1}.__md_methods);", + type.GetAssemblyQualifiedName (codeGenState.TypeCache), + javaKey + ); + } } + regCallsWriter.Close (); - // Step 3 - Merge [Activity] and friends into AndroidManifest.xml + var real_app_dir = Path.Combine (OutputDirectory, "src", "mono", "android", "app"); + string applicationTemplateFile = "ApplicationRegistration.java"; + SaveResource ( + applicationTemplateFile, + applicationTemplateFile, + real_app_dir, + template => template.Replace ("// REGISTER_APPLICATION_AND_INSTRUMENTATION_CLASSES_HERE", regCallsWriter.ToString ()) + ); + } + + IList MergeManifest (NativeCodeGenState codeGenState, Dictionary userAssemblies) + { var manifest = new ManifestDocument (ManifestTemplate) { PackageName = PackageName, VersionName = VersionName, ApplicationLabel = ApplicationLabel ?? PackageName, Placeholders = ManifestPlaceholders, - Resolver = res, + Resolver = codeGenState.Resolver, SdkDir = AndroidSdkDir, TargetSdkVersion = AndroidSdkPlatform, MinSdkVersion = MonoAndroidHelper.ConvertSupportedOSPlatformVersionToApiLevel (SupportedOSPlatformVersion).ToString (), @@ -342,7 +330,7 @@ void Run (XAAssemblyResolver res, bool useMarshalMethods) } else if (!string.IsNullOrEmpty (VersionCode)) { manifest.VersionCode = VersionCode; } - manifest.Assemblies.AddRange (userAssemblies.Values); + manifest.Assemblies.AddRange (userAssemblies.Values.Select (item => item.ItemSpec)); if (!String.IsNullOrWhiteSpace (CheckedBuild)) { // We don't validate CheckedBuild value here, this will be done in BuildApk. We just know that if it's @@ -351,216 +339,67 @@ void Run (XAAssemblyResolver res, bool useMarshalMethods) manifest.ForceExtractNativeLibs = true; } - var additionalProviders = manifest.Merge (Log, cache, allJavaTypes, ApplicationJavaClass, EmbedAssemblies, BundledWearApplicationName, MergedManifestDocuments); + IList additionalProviders = manifest.Merge (Log, codeGenState.TypeCache, codeGenState.AllJavaTypes, ApplicationJavaClass, EmbedAssemblies, BundledWearApplicationName, MergedManifestDocuments); // Only write the new manifest if it actually changed if (manifest.SaveIfChanged (Log, MergedAndroidManifestOutput)) { Log.LogDebugMessage ($"Saving: {MergedAndroidManifestOutput}"); } - // Create additional runtime provider java sources. - string providerTemplateFile = "MonoRuntimeProvider.Bundled.java"; - string providerTemplate = GetResource (providerTemplateFile); - - foreach (var provider in additionalProviders) { - var contents = providerTemplate.Replace ("MonoRuntimeProvider", provider); - var real_provider = Path.Combine (OutputDirectory, "src", "mono", provider + ".java"); - Files.CopyIfStringChanged (contents, real_provider); - } - - // Create additional application java sources. - StringWriter regCallsWriter = new StringWriter (); - regCallsWriter.WriteLine ("\t\t// Application and Instrumentation ACWs must be registered first."); - foreach (JavaType jt in javaTypes) { - TypeDefinition type = jt.Type; - if (JavaNativeTypeManager.IsApplication (type, cache) || JavaNativeTypeManager.IsInstrumentation (type, cache)) { - if (classifier != null && !classifier.FoundDynamicallyRegisteredMethods (type)) { - continue; - } - - string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'); - regCallsWriter.WriteLine ("\t\tmono.android.Runtime.register (\"{0}\", {1}.class, {1}.__md_methods);", - type.GetAssemblyQualifiedName (cache), javaKey); - } - } - regCallsWriter.Close (); - - var real_app_dir = Path.Combine (OutputDirectory, "src", "mono", "android", "app"); - string applicationTemplateFile = "ApplicationRegistration.java"; - SaveResource (applicationTemplateFile, applicationTemplateFile, real_app_dir, - template => template.Replace ("// REGISTER_APPLICATION_AND_INSTRUMENTATION_CLASSES_HERE", regCallsWriter.ToString ())); - - if (useMarshalMethods) { - classifier.AddSpecialCaseMethods (); - - Log.LogDebugMessage ($"Number of generated marshal methods: {classifier.MarshalMethods.Count}"); - - if (classifier.RejectedMethodCount > 0) { - Log.LogWarning ($"Number of methods in the project that will be registered dynamically: {classifier.RejectedMethodCount}"); - } - - if (classifier.WrappedMethodCount > 0) { - // TODO: change to LogWarning once the generator can output code which requires no non-blittable wrappers - Log.LogDebugMessage ($"Number of methods in the project that need marshal method wrappers: {classifier.WrappedMethodCount}"); - } - } - - void MaybeAddAbiSpecifcAssembly (ITaskItem assembly, string fileName) - { - if (abiSpecificAssembliesByPath == null) { - return; - } - - string? abi = assembly.GetMetadata ("Abi"); - if (!String.IsNullOrEmpty (abi)) { - if (!abiSpecificAssembliesByPath.TryGetValue (fileName, out List? items)) { - items = new List (); - abiSpecificAssembliesByPath.Add (fileName, items); - } - - items.Add (assembly); - } - } + return additionalProviders; } - AssemblyDefinition LoadAssembly (string path, XAAssemblyResolver? resolver = null) + (bool success, NativeCodeGenState? stubsState) GenerateJavaSourcesAndMaybeClassifyMarshalMethods (AndroidTargetArch arch, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods, bool generateJavaCode) { - string pdbPath = Path.ChangeExtension (path, ".pdb"); - var readerParameters = new ReaderParameters { - AssemblyResolver = resolver, - InMemory = false, - ReadingMode = ReadingMode.Immediate, - ReadSymbols = File.Exists (pdbPath), - ReadWrite = false, - }; - - MemoryMappedViewStream? viewStream = null; - try { - // Create stream because CreateFromFile(string, ...) uses FileShare.None which is too strict - using var fileStream = new FileStream (path, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, false); - using var mappedFile = MemoryMappedFile.CreateFromFile ( - fileStream, null, fileStream.Length, MemoryMappedFileAccess.Read, HandleInheritability.None, true); - viewStream = mappedFile.CreateViewStream (0, 0, MemoryMappedFileAccess.Read); + XAAssemblyResolver resolver = MakeResolver (useMarshalMethods, arch, assemblies); + var tdCache = new TypeDefinitionCache (); + (List allJavaTypes, List javaTypesForJCW) = ScanForJavaTypes (resolver, tdCache, assemblies, userAssemblies, useMarshalMethods); + var jcwContext = new JCWGeneratorContext (arch, resolver, assemblies.Values, javaTypesForJCW, tdCache, useMarshalMethods); + var jcwGenerator = new JCWGenerator (Log, jcwContext); + bool success; - AssemblyDefinition result = ModuleDefinition.ReadModule (viewStream, readerParameters).Assembly; - - // We transferred the ownership of the viewStream to the collection. - viewStream = null; - - return result; - } finally { - viewStream?.Dispose (); + if (generateJavaCode) { + success = jcwGenerator.GenerateAndClassify (AndroidSdkPlatform, outputPath: Path.Combine (OutputDirectory, "src"), ApplicationJavaClass); + } else { + success = jcwGenerator.Classify (AndroidSdkPlatform); } - } - bool CreateJavaSources (IEnumerable newJavaTypes, TypeDefinitionCache cache, MarshalMethodsClassifier classifier, bool useMarshalMethods) - { - if (useMarshalMethods && classifier == null) { - throw new ArgumentNullException (nameof (classifier)); + if (!success) { + return (false, null); } - string outputPath = Path.Combine (OutputDirectory, "src"); - string monoInit = GetMonoInitSource (AndroidSdkPlatform); - bool hasExportReference = ResolvedAssemblies.Any (assembly => Path.GetFileName (assembly.ItemSpec) == "Mono.Android.Export.dll"); - bool generateOnCreateOverrides = int.Parse (AndroidSdkPlatform) <= 10; + return (true, new NativeCodeGenState (arch, tdCache, resolver, allJavaTypes, javaTypesForJCW, jcwGenerator.Classifier)); + } - var reader_options = new CallableWrapperReaderOptions { - DefaultApplicationJavaClass = ApplicationJavaClass, - DefaultGenerateOnCreateOverrides = generateOnCreateOverrides, - DefaultMonoRuntimeInitialization = monoInit, - MethodClassifier = classifier, - }; - var writer_options = new CallableWrapperWriterOptions { - CodeGenerationTarget = JavaPeerStyle.XAJavaInterop1 + (List allJavaTypes, List javaTypesForJCW) ScanForJavaTypes (XAAssemblyResolver res, TypeDefinitionCache cache, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods) + { + var scanner = new XAJavaTypeScanner (res.TargetArch, Log, cache) { + ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, }; + List allJavaTypes = scanner.GetJavaTypes (assemblies.Values, res); + var javaTypesForJCW = new List (); - bool ok = true; - foreach (JavaType jt in newJavaTypes) { - TypeDefinition t = jt.Type; // JCW generator doesn't care about ABI-specific types or token ids - if (t.IsInterface) { - // Interfaces are in typemap but they shouldn't have JCW generated for them + foreach (TypeDefinition type in allJavaTypes) { + // When marshal methods are in use we do not want to skip non-user assemblies (such as Mono.Android) - we need to generate JCWs for them during + // application build, unlike in Debug configuration or when marshal methods are disabled, in which case we use JCWs generated during Xamarin.Android + // build and stored in a jar file. + if ((!useMarshalMethods && !userAssemblies.ContainsKey (type.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (type, cache)) { continue; } - - using (var writer = MemoryStreamPool.Shared.CreateStreamWriter ()) { - try { - var jcw_type = CecilImporter.CreateType (t, cache, reader_options); - - jcw_type.Generate (writer, writer_options); - - if (useMarshalMethods) { - if (classifier.FoundDynamicallyRegisteredMethods (t)) { - Log.LogWarning ($"Type '{t.GetAssemblyQualifiedName (cache)}' will register some of its Java override methods dynamically. This may adversely affect runtime performance. See preceding warnings for names of dynamically registered methods."); - } - } - - writer.Flush (); - - var path = jcw_type.GetDestinationPath (outputPath); - Files.CopyIfStreamChanged (writer.BaseStream, path); - if (jcw_type.HasExport && !hasExportReference) - Diagnostic.Error (4210, Properties.Resources.XA4210); - } catch (XamarinAndroidException xae) { - ok = false; - Log.LogError ( - subcategory: "", - errorCode: "XA" + xae.Code, - helpKeyword: string.Empty, - file: xae.SourceFile, - lineNumber: xae.SourceLine, - columnNumber: 0, - endLineNumber: 0, - endColumnNumber: 0, - message: xae.MessageWithoutCode, - messageArgs: Array.Empty () - ); - } catch (DirectoryNotFoundException ex) { - ok = false; - if (OS.IsWindows) { - Diagnostic.Error (5301, Properties.Resources.XA5301, t.FullName, ex); - } else { - Diagnostic.Error (4209, Properties.Resources.XA4209, t.FullName, ex); - } - } catch (Exception ex) { - ok = false; - Diagnostic.Error (4209, Properties.Resources.XA4209, t.FullName, ex); - } - } - } - - if (useMarshalMethods) { - BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (MarshalMethodsRegisterTaskKey), new MarshalMethodsState (classifier.MarshalMethods), RegisteredTaskObjectLifetime.Build); + javaTypesForJCW.Add (type); } - return ok; + return (allJavaTypes, javaTypesForJCW); } - static string GetMonoInitSource (string androidSdkPlatform) + void RewriteMarshalMethods (NativeCodeGenState state, bool brokenExceptionTransitionsEnabled) { - // Lookup the mono init section from MonoRuntimeProvider: - // Mono Runtime Initialization {{{ - // }}} - var builder = new StringBuilder (); - var runtime = "Bundled"; - var api = ""; - if (int.TryParse (androidSdkPlatform, out int apiLevel) && apiLevel < 21) { - api = ".20"; - } - var assembly = Assembly.GetExecutingAssembly (); - using (var s = assembly.GetManifestResourceStream ($"MonoRuntimeProvider.{runtime}{api}.java")) - using (var reader = new StreamReader (s)) { - bool copy = false; - string line; - while ((line = reader.ReadLine ()) != null) { - if (string.CompareOrdinal ("\t\t// Mono Runtime Initialization {{{", line) == 0) - copy = true; - if (copy) - builder.AppendLine (line); - if (string.CompareOrdinal ("\t\t// }}}", line) == 0) - break; - } + if (state.Classifier == null) { + return; } - return builder.ToString (); + + var rewriter = new MarshalMethodsAssemblyRewriter (Log, state.TargetArch, state.Classifier, state.Resolver); + rewriter.Rewrite (brokenExceptionTransitionsEnabled); } string GetResource (string resource) @@ -577,146 +416,26 @@ void SaveResource (string resource, string filename, string destDir, Func types, TypeDefinitionCache cache) + void WriteTypeMappings (NativeCodeGenState state) { - var tmg = new TypeMapGenerator (Log, SupportedAbis); - if (!tmg.Generate (Debug, SkipJniAddNativeMethodRegistrationAttributeScan, types, cache, TypemapOutputDirectory, GenerateNativeAssembly, out ApplicationConfigTaskState appConfState)) { + Log.LogDebugMessage ($"Generating type maps for architecture '{state.TargetArch}'"); + var tmg = new TypeMapGenerator (Log, state); + if (!tmg.Generate (Debug, SkipJniAddNativeMethodRegistrationAttributeScan, TypemapOutputDirectory, GenerateNativeAssembly)) { throw new XamarinAndroidException (4308, Properties.Resources.XA4308); } - GeneratedBinaryTypeMaps = tmg.GeneratedBinaryTypeMaps.ToArray (); - BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (ApplicationConfigTaskState.RegisterTaskObjectKey), appConfState, RegisteredTaskObjectLifetime.Build); - } - - /// - /// - /// Classifier will see only unique assemblies, since that's what's processed by the JI type scanner - even though some assemblies may have - /// abi-specific features (e.g. inlined `IntPtr.Size` or processor-specific intrinsics), the **types** and **methods** will all be the same and, thus, - /// there's no point in scanning all of the additional copies of the same assembly. - /// - /// - /// This, however, doesn't work for the rewriter which needs to rewrite all of the copies so that they all have the same generated wrappers. In - /// order to do that, we need to go over the list of assemblies found by the classifier, see if they are abi-specific ones and then add all the - /// marshal methods from the abi-specific assembly copies, so that the rewriter can easily rewrite them all. - /// - /// - /// This method returns a dictionary matching `AssemblyDefinition` instances to the path on disk to the assembly file they were loaded from. It is necessary - /// because uses a stream to load the data, in order to avoid later sharing violation issues when writing the assemblies. Path - /// information is required by to be available for each - /// - /// - Dictionary AddMethodsFromAbiSpecificAssemblies (MarshalMethodsClassifier classifier, XAAssemblyResolver resolver, Dictionary> abiSpecificAssemblies) - { - IDictionary> marshalMethods = classifier.MarshalMethods; - ICollection assemblies = classifier.Assemblies; - var newAssemblies = new List (); - var assemblyPaths = new Dictionary (); - - foreach (AssemblyDefinition asmdef in assemblies) { - string fileName = Path.GetFileName (asmdef.MainModule.FileName); - if (!abiSpecificAssemblies.TryGetValue (fileName, out List? abiAssemblyItems)) { - continue; - } - - List assemblyMarshalMethods = FindMarshalMethodsForAssembly (marshalMethods, asmdef);; - Log.LogDebugMessage ($"Assembly {fileName} is ABI-specific"); - foreach (ITaskItem abiAssemblyItem in abiAssemblyItems) { - if (String.Compare (abiAssemblyItem.ItemSpec, asmdef.MainModule.FileName, StringComparison.Ordinal) == 0) { - continue; - } - - Log.LogDebugMessage ($"Looking for matching mashal methods in {abiAssemblyItem.ItemSpec}"); - FindMatchingMethodsInAssembly (abiAssemblyItem, classifier, assemblyMarshalMethods, resolver, newAssemblies, assemblyPaths); - } - } - if (newAssemblies.Count > 0) { - foreach (AssemblyDefinition asmdef in newAssemblies) { - assemblies.Add (asmdef); - } + string abi = MonoAndroidHelper.ArchToAbi (state.TargetArch); + var items = new List (); + foreach (string file in tmg.GeneratedBinaryTypeMaps) { + var item = new TaskItem (file); + string fileName = Path.GetFileName (file); + item.SetMetadata ("DestinationSubPath", $"{abi}/{fileName}"); + item.SetMetadata ("DestinationSubDirectory", $"{abi}/"); + item.SetMetadata ("Abi", abi); + items.Add (item); } - return assemblyPaths; - } - - List FindMarshalMethodsForAssembly (IDictionary> marshalMethods, AssemblyDefinition asm) - { - var seenNativeCallbacks = new HashSet (); - var assemblyMarshalMethods = new List (); - - foreach (var kvp in marshalMethods) { - foreach (MarshalMethodEntry method in kvp.Value) { - if (method.NativeCallback.Module.Assembly != asm) { - continue; - } - - // More than one overriden method can use the same native callback method, we're interested only in unique native - // callbacks, since that's what gets rewritten. - if (seenNativeCallbacks.Contains (method.NativeCallback)) { - continue; - } - - seenNativeCallbacks.Add (method.NativeCallback); - assemblyMarshalMethods.Add (method); - } - } - - return assemblyMarshalMethods; - } - - void FindMatchingMethodsInAssembly (ITaskItem assemblyItem, MarshalMethodsClassifier classifier, List assemblyMarshalMethods, XAAssemblyResolver resolver, List newAssemblies, Dictionary assemblyPaths) - { - AssemblyDefinition asm = LoadAssembly (assemblyItem.ItemSpec, resolver); - newAssemblies.Add (asm); - assemblyPaths.Add (asm, assemblyItem.ItemSpec); - - foreach (MarshalMethodEntry methodEntry in assemblyMarshalMethods) { - TypeDefinition wantedType = methodEntry.NativeCallback.DeclaringType; - TypeDefinition? type = asm.MainModule.FindType (wantedType.FullName); - if (type == null) { - throw new InvalidOperationException ($"Internal error: type '{wantedType.FullName}' not found in assembly '{assemblyItem.ItemSpec}', a linker error?"); - } - - if (type.MetadataToken != wantedType.MetadataToken) { - throw new InvalidOperationException ($"Internal error: type '{type.FullName}' in assembly '{assemblyItem.ItemSpec}' has a different token ID than the original type"); - } - - FindMatchingMethodInType (methodEntry, type, classifier); - } - } - - void FindMatchingMethodInType (MarshalMethodEntry methodEntry, TypeDefinition type, MarshalMethodsClassifier classifier) - { - string callbackName = methodEntry.NativeCallback.FullName; - - foreach (MethodDefinition typeNativeCallbackMethod in type.Methods) { - if (String.Compare (typeNativeCallbackMethod.FullName, callbackName, StringComparison.Ordinal) != 0) { - continue; - } - - if (typeNativeCallbackMethod.Parameters.Count != methodEntry.NativeCallback.Parameters.Count) { - continue; - } - - if (typeNativeCallbackMethod.MetadataToken != methodEntry.NativeCallback.MetadataToken) { - throw new InvalidOperationException ($"Internal error: tokens don't match for '{typeNativeCallbackMethod.FullName}'"); - } - - bool allMatch = true; - for (int i = 0; i < typeNativeCallbackMethod.Parameters.Count; i++) { - if (String.Compare (typeNativeCallbackMethod.Parameters[i].ParameterType.FullName, methodEntry.NativeCallback.Parameters[i].ParameterType.FullName, StringComparison.Ordinal) != 0) { - allMatch = false; - break; - } - } - - if (!allMatch) { - continue; - } - - Log.LogDebugMessage ($"Found match for '{typeNativeCallbackMethod.FullName}' in {type.Module.FileName}"); - string methodKey = classifier.GetStoreMethodKey (methodEntry); - classifier.MarshalMethods[methodKey].Add (new MarshalMethodEntry (methodEntry, typeNativeCallbackMethod)); - } + GeneratedBinaryTypeMaps = items.ToArray (); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 101f70f84c6..2f3e7bed8ef 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -144,25 +144,7 @@ public override bool RunTask () return !Log.HasLoggedErrors; } - static internal AndroidTargetArch GetAndroidTargetArchForAbi (string abi) - { - switch (abi.Trim ()) { - case "armeabi-v7a": - return AndroidTargetArch.Arm; - - case "arm64-v8a": - return AndroidTargetArch.Arm64; - - case "x86": - return AndroidTargetArch.X86; - - case "x86_64": - return AndroidTargetArch.X86_64; - - default: - throw new InvalidOperationException ($"Unknown ABI {abi}"); - } - } + static internal AndroidTargetArch GetAndroidTargetArchForAbi (string abi) => MonoAndroidHelper.AbiToTargetArch (abi); static readonly string[] defaultLogLevel = {"MONO_LOG_LEVEL", "info"}; static readonly string[] defaultMonoDebug = {"MONO_DEBUG", "gen-compact-seq-points"}; @@ -254,32 +236,26 @@ void AddEnvironment () HashSet archAssemblyNames = null; HashSet uniqueAssemblyNames = new HashSet (StringComparer.OrdinalIgnoreCase); Action updateAssemblyCount = (ITaskItem assembly) => { - // We need to use the 'RelativePath' metadata, if found, because it will give us the correct path for satellite assemblies - with the culture in the path. - string? relativePath = assembly.GetMetadata ("RelativePath"); - string assemblyName = String.IsNullOrEmpty (relativePath) ? Path.GetFileName (assembly.ItemSpec) : relativePath; - if (!uniqueAssemblyNames.Contains (assemblyName)) { - uniqueAssemblyNames.Add (assemblyName); - } + string? culture = assembly.GetMetadata ("Culture"); + string fileName = Path.GetFileName (assembly.ItemSpec); + string assemblyName; - if (!UseAssemblyStore) { - assemblyCount++; - return; + if (String.IsNullOrEmpty (culture)) { + assemblyName = fileName; + } else { + assemblyName = $"{culture}/{fileName}"; } - if (Boolean.TryParse (assembly.GetMetadata ("AndroidSkipAddToPackage"), out bool value) && value) { - return; + if (!uniqueAssemblyNames.Contains (assemblyName)) { + uniqueAssemblyNames.Add (assemblyName); } - string abi = assembly.GetMetadata ("Abi"); - if (String.IsNullOrEmpty (abi)) { - assemblyCount++; - } else { - archAssemblyNames ??= new HashSet (StringComparer.OrdinalIgnoreCase); + string abi = MonoAndroidHelper.GetAssemblyAbi (assembly); + archAssemblyNames ??= new HashSet (StringComparer.OrdinalIgnoreCase); - if (!archAssemblyNames.Contains (assemblyName)) { - assemblyCount++; - archAssemblyNames.Add (assemblyName); - } + if (!archAssemblyNames.Contains (assemblyName)) { + assemblyCount++; + archAssemblyNames.Add (assemblyName); } }; @@ -347,8 +323,15 @@ void AddEnvironment () } } + Dictionary? nativeCodeGenStates = null; + if (enableMarshalMethods) { + nativeCodeGenStates = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal> ( + ProjectSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateRegisterTaskKey), + RegisteredTaskObjectLifetime.Build + ); + } + bool haveRuntimeConfigBlob = !String.IsNullOrEmpty (RuntimeConfigBinFilePath) && File.Exists (RuntimeConfigBinFilePath); - var appConfState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (ApplicationConfigTaskState.RegisterTaskObjectKey), RegisteredTaskObjectLifetime.Build); var jniRemappingNativeCodeInfo = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (GenerateJniRemappingNativeCode.JniRemappingNativeCodeInfoKey), RegisteredTaskObjectLifetime.Build); var appConfigAsmGen = new ApplicationConfigNativeAssemblyGenerator (environmentVariables, systemProperties, Log) { UsesMonoAOT = usesMonoAOT, @@ -361,14 +344,10 @@ void AddEnvironment () PackageNamingPolicy = pnp, BoundExceptionType = boundExceptionType, InstantRunEnabled = InstantRunEnabled, - JniAddNativeMethodRegistrationAttributePresent = appConfState != null ? appConfState.JniAddNativeMethodRegistrationAttributePresent : false, + JniAddNativeMethodRegistrationAttributePresent = NativeCodeGenState.Template != null ? NativeCodeGenState.Template.JniAddNativeMethodRegistrationAttributePresent : false, HaveRuntimeConfigBlob = haveRuntimeConfigBlob, NumberOfAssembliesInApk = assemblyCount, BundledAssemblyNameWidth = assemblyNameWidth, - NumberOfAssemblyStoresInApks = 2, // Until feature APKs are a thing, we're going to have just two stores in each app - one for arch-agnostic - // and up to 4 other for arch-specific assemblies. Only **one** arch-specific store is ever loaded on the app - // runtime, thus the number 2 here. All architecture specific stores contain assemblies with the same names - // and in the same order. MonoComponents = (MonoComponent)monoComponents, NativeLibraries = uniqueNativeLibraries, HaveAssemblyStore = UseAssemblyStore, @@ -381,21 +360,6 @@ void AddEnvironment () }; LLVMIR.LlvmIrModule appConfigModule = appConfigAsmGen.Construct (); - var marshalMethodsState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (GenerateJavaStubs.MarshalMethodsRegisterTaskKey), RegisteredTaskObjectLifetime.Build); - MarshalMethodsNativeAssemblyGenerator marshalMethodsAsmGen; - - if (enableMarshalMethods) { - marshalMethodsAsmGen = new MarshalMethodsNativeAssemblyGenerator ( - Log, - assemblyCount, - uniqueAssemblyNames, - marshalMethodsState?.MarshalMethods - ); - } else { - marshalMethodsAsmGen = new MarshalMethodsNativeAssemblyGenerator (Log, assemblyCount, uniqueAssemblyNames); - } - LLVMIR.LlvmIrModule marshalMethodsModule = marshalMethodsAsmGen.Construct (); - foreach (string abi in SupportedAbis) { string targetAbi = abi.ToLowerInvariant (); string environmentBaseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"environment.{targetAbi}"); @@ -404,29 +368,54 @@ void AddEnvironment () string marshalMethodsLlFilePath = $"{marshalMethodsBaseAsmFilePath}.ll"; AndroidTargetArch targetArch = GetAndroidTargetArchForAbi (abi); - using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { - try { - appConfigAsmGen.Generate (appConfigModule, targetArch, sw, environmentLlFilePath); - } catch { - throw; - } finally { - sw.Flush (); - Files.CopyIfStreamChanged (sw.BaseStream, environmentLlFilePath); - } + using var appConfigWriter = MemoryStreamPool.Shared.CreateStreamWriter (); + try { + appConfigAsmGen.Generate (appConfigModule, targetArch, appConfigWriter, environmentLlFilePath); + } catch { + throw; + } finally { + appConfigWriter.Flush (); + Files.CopyIfStreamChanged (appConfigWriter.BaseStream, environmentLlFilePath); } - using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { - try { - marshalMethodsAsmGen.Generate (marshalMethodsModule, targetArch, sw, marshalMethodsLlFilePath); - } catch { - throw; - } finally { - sw.Flush (); - Files.CopyIfStreamChanged (sw.BaseStream, marshalMethodsLlFilePath); - } + MarshalMethodsNativeAssemblyGenerator marshalMethodsAsmGen; + if (enableMarshalMethods) { + marshalMethodsAsmGen = new MarshalMethodsNativeAssemblyGenerator ( + Log, + assemblyCount, + uniqueAssemblyNames, + EnsureCodeGenState (targetArch) + ); + } else { + marshalMethodsAsmGen = new MarshalMethodsNativeAssemblyGenerator ( + Log, + targetArch, + assemblyCount, + uniqueAssemblyNames + ); + } + + LLVMIR.LlvmIrModule marshalMethodsModule = marshalMethodsAsmGen.Construct (); + using var marshalMethodsWriter = MemoryStreamPool.Shared.CreateStreamWriter (); + try { + marshalMethodsAsmGen.Generate (marshalMethodsModule, targetArch, marshalMethodsWriter, marshalMethodsLlFilePath); + } catch { + throw; + } finally { + marshalMethodsWriter.Flush (); + Files.CopyIfStreamChanged (marshalMethodsWriter.BaseStream, marshalMethodsLlFilePath); } } + NativeCodeGenState EnsureCodeGenState (AndroidTargetArch targetArch) + { + if (nativeCodeGenStates == null || !nativeCodeGenStates.TryGetValue (targetArch, out NativeCodeGenState? state)) { + throw new InvalidOperationException ($"Internal error: missing native code generation state for architecture '{targetArch}'"); + } + + return state; + } + void AddEnvironmentVariable (string name, string value) { if (Char.IsUpper(name [0]) || !Char.IsLetter(name [0])) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/LinkAssembliesNoShrink.cs b/src/Xamarin.Android.Build.Tasks/Tasks/LinkAssembliesNoShrink.cs index 0533766aa12..bb7d7c54516 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/LinkAssembliesNoShrink.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/LinkAssembliesNoShrink.cs @@ -1,4 +1,6 @@ #nullable enable +using System.Collections.Generic; + using Java.Interop.Tools.Cecil; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; @@ -6,6 +8,7 @@ using System; using System.IO; using Microsoft.Android.Build.Tasks; +using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks { @@ -14,6 +17,15 @@ namespace Xamarin.Android.Tasks /// public class LinkAssembliesNoShrink : AndroidTask { + sealed class RunState + { + public DirectoryAssemblyResolver? resolver = null; + public TypeDefinitionCache? cache = null; + public FixAbstractMethodsStep? fixAbstractMethodsStep = null; + public AddKeepAlivesStep? addKeepAliveStep = null; + public FixLegacyResourceDesignerStep? fixLegacyResourceDesignerStep = null; + } + public override string TaskPrefix => "LNS"; /// @@ -57,56 +69,91 @@ public override bool RunTask () DeterministicMvid = Deterministic, }; - using (var resolver = new DirectoryAssemblyResolver (this.CreateTaskLogger (), loadDebugSymbols: ReadSymbols, loadReaderParameters: readerParameters)) { - // Add SearchDirectories with ResolvedAssemblies - foreach (var assembly in ResolvedAssemblies) { - var path = Path.GetFullPath (Path.GetDirectoryName (assembly.ItemSpec)); - if (!resolver.SearchDirectories.Contains (path)) - resolver.SearchDirectories.Add (path); - } + Dictionary> perArchAssemblies = MonoAndroidHelper.GetPerArchAssemblies (ResolvedAssemblies, Array.Empty (), validate: false); + var runState = new RunState (); + AndroidTargetArch currentArch = AndroidTargetArch.None; - // Set up the FixAbstractMethodsStep and AddKeepAlivesStep - var cache = new TypeDefinitionCache (); - var fixAbstractMethodsStep = new FixAbstractMethodsStep (resolver, cache, Log); - var addKeepAliveStep = new AddKeepAlivesStep (resolver, cache, Log); - var fixLegacyResourceDesignerStep = new FixLegacyResourceDesignerStep (resolver, cache, Log); - for (int i = 0; i < SourceFiles.Length; i++) { - var source = SourceFiles [i]; - var destination = DestinationFiles [i]; - var assemblyName = Path.GetFileNameWithoutExtension (source.ItemSpec); - - // In .NET 6+, we can skip the main assembly - if (!AddKeepAlives && assemblyName == TargetName) { - CopyIfChanged (source, destination); - continue; - } - if (fixAbstractMethodsStep.IsProductOrSdkAssembly (assemblyName)) { - CopyIfChanged (source, destination); - continue; - } + for (int i = 0; i < SourceFiles.Length; i++) { + ITaskItem source = SourceFiles [i]; + AndroidTargetArch sourceArch = GetValidArchitecture (source); + ITaskItem destination = DestinationFiles [i]; + AndroidTargetArch destinationArch = GetValidArchitecture (destination); - // Only run the step on "MonoAndroid" assemblies - if (MonoAndroidHelper.IsMonoAndroidAssembly (source) && !MonoAndroidHelper.IsSharedRuntimeAssembly (source.ItemSpec)) { - var assemblyDefinition = resolver.GetAssembly (source.ItemSpec); - - bool save = fixAbstractMethodsStep.FixAbstractMethods (assemblyDefinition); - if (UseDesignerAssembly) - save |= fixLegacyResourceDesignerStep.ProcessAssemblyDesigner (assemblyDefinition); - if (AddKeepAlives) - save |= addKeepAliveStep.AddKeepAlives (assemblyDefinition); - if (save) { - Log.LogDebugMessage ($"Saving modified assembly: {destination.ItemSpec}"); - writerParameters.WriteSymbols = assemblyDefinition.MainModule.HasSymbols; - assemblyDefinition.Write (destination.ItemSpec, writerParameters); - continue; + if (sourceArch != destinationArch) { + throw new InvalidOperationException ($"Internal error: assembly '{sourceArch}' targets architecture '{sourceArch}', while destination assembly '{destination}' targets '{destinationArch}' instead"); + } + + // Each architecture must have a different set of context classes, or otherwise only the first instance of the assembly may be rewritten. + if (currentArch != sourceArch) { + currentArch = sourceArch; + runState.resolver?.Dispose (); + runState.resolver = new DirectoryAssemblyResolver (this.CreateTaskLogger (), loadDebugSymbols: ReadSymbols, loadReaderParameters: readerParameters); + + // Add SearchDirectories for the current architecture's ResolvedAssemblies + foreach (var kvp in perArchAssemblies[sourceArch]) { + ITaskItem assembly = kvp.Value; + var path = Path.GetFullPath (Path.GetDirectoryName (assembly.ItemSpec)); + if (!runState.resolver.SearchDirectories.Contains (path)) { + runState.resolver.SearchDirectories.Add (path); } } - CopyIfChanged (source, destination); + // Set up the FixAbstractMethodsStep and AddKeepAlivesStep + runState.cache = new TypeDefinitionCache (); + runState.fixAbstractMethodsStep = new FixAbstractMethodsStep (runState.resolver, runState.cache, Log); + runState.addKeepAliveStep = new AddKeepAlivesStep (runState.resolver, runState.cache, Log); + runState.fixLegacyResourceDesignerStep = new FixLegacyResourceDesignerStep (runState.resolver, runState.cache, Log); } - } + DoRunTask (source, destination, runState, writerParameters); + } + runState.resolver?.Dispose (); return !Log.HasLoggedErrors; + + AndroidTargetArch GetValidArchitecture (ITaskItem item) + { + AndroidTargetArch ret = MonoAndroidHelper.GetTargetArch (item); + if (ret == AndroidTargetArch.None) { + throw new InvalidOperationException ($"Internal error: assembly '{item}' doesn't target any architecture."); + } + + return ret; + } + } + + void DoRunTask (ITaskItem source, ITaskItem destination, RunState runState, WriterParameters writerParameters) + { + var assemblyName = Path.GetFileNameWithoutExtension (source.ItemSpec); + + // In .NET 6+, we can skip the main assembly + if (!AddKeepAlives && assemblyName == TargetName) { + CopyIfChanged (source, destination); + return; + } + if (runState.fixAbstractMethodsStep!.IsProductOrSdkAssembly (assemblyName)) { + CopyIfChanged (source, destination); + return; + } + + // Only run the step on "MonoAndroid" assemblies + if (MonoAndroidHelper.IsMonoAndroidAssembly (source) && !MonoAndroidHelper.IsSharedRuntimeAssembly (source.ItemSpec)) { + AssemblyDefinition assemblyDefinition = runState.resolver!.GetAssembly (source.ItemSpec); + + bool save = runState.fixAbstractMethodsStep.FixAbstractMethods (assemblyDefinition); + if (UseDesignerAssembly) + save |= runState.fixLegacyResourceDesignerStep!.ProcessAssemblyDesigner (assemblyDefinition); + if (AddKeepAlives) + save |= runState.addKeepAliveStep!.AddKeepAlives (assemblyDefinition); + if (save) { + Log.LogDebugMessage ($"Saving modified assembly: {destination.ItemSpec}"); + Directory.CreateDirectory (Path.GetDirectoryName (destination.ItemSpec)); + writerParameters.WriteSymbols = assemblyDefinition.MainModule.HasSymbols; + assemblyDefinition.Write (destination.ItemSpec, writerParameters); + return; + } + } + + CopyIfChanged (source, destination); } void CopyIfChanged (ITaskItem source, ITaskItem destination) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareSatelliteAssemblies.cs b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareSatelliteAssemblies.cs new file mode 100644 index 00000000000..8ba0829c01c --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareSatelliteAssemblies.cs @@ -0,0 +1,71 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Microsoft.Android.Build.Tasks; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +public class PrepareSatelliteAssemblies : AndroidTask +{ + public override string TaskPrefix => "PSA"; + + [Required] + public string[] BuildTargetAbis { get; set; } = Array.Empty (); + + [Required] + public ITaskItem[] ReferenceSatellitePaths { get; set; } = Array.Empty (); + + [Required] + public ITaskItem[] IntermediateSatelliteAssemblies { get; set; } = Array.Empty (); + + [Output] + public ITaskItem[] ProcessedSatelliteAssemblies { get; set; } + + public override bool RunTask () + { + var output = new List (); + + SetMetadata (ReferenceSatellitePaths, output); + SetMetadata (IntermediateSatelliteAssemblies, output); + + ProcessedSatelliteAssemblies = output.ToArray (); + return !Log.HasLoggedErrors; + } + + void SetMetadata (ITaskItem[] items, List output) + { + foreach (ITaskItem item in items) { + SetMetadata (item, output); + } + } + + void SetMetadata (ITaskItem item, List output) + { + string? culture = item.GetMetadata ("Culture"); + if (String.IsNullOrEmpty (culture)) { + if (!SatelliteAssembly.TryGetSatelliteCultureAndFileName (item.ItemSpec, out culture, out _)) { + throw new InvalidOperationException ($"Assembly item '{item}' is missing the 'Culture' metadata and it wasn't possible to obtain it from the path"); + } + item.SetMetadata ("Culture", culture); + } + + string assemblyName = Path.GetFileName (item.ItemSpec); + foreach (string abi in BuildTargetAbis) { + var newItem = new TaskItem (item); + newItem.SetMetadata ("Abi", abi); + + SetDestinationPathsMetadata (newItem, MonoAndroidHelper.MakeZipArchivePath (abi, culture, assemblyName)); + output.Add (newItem); + } + + void SetDestinationPathsMetadata (ITaskItem item, string zipArchivePath) + { + item.SetMetadata ("DestinationSubPath", zipArchivePath); + item.SetMetadata ("DestinationSubDirectory", Path.GetDirectoryName (zipArchivePath) + Path.DirectorySeparatorChar); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs index 80d16a1f72c..98065ecf653 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs @@ -26,6 +26,8 @@ public class ProcessAssemblies : AndroidTask [Required] public string [] RuntimeIdentifiers { get; set; } = Array.Empty(); + public bool DesignTimeBuild { get; set; } + public bool AndroidIncludeDebugSymbols { get; set; } public bool PublishTrimmed { get; set; } @@ -57,15 +59,7 @@ public override bool RunTask () } } - // We only need to "dedup" assemblies when there is more than one RID - if (RuntimeIdentifiers.Length > 1) { - Log.LogDebugMessage ("Deduplicating assemblies per RuntimeIdentifier"); - DeduplicateAssemblies (output, symbols); - } else { - Log.LogDebugMessage ("Found a single RuntimeIdentifier"); - SetMetadataForAssemblies (output, symbols); - } - + SetMetadataForAssemblies (output, symbols); OutputAssemblies = output.ToArray (); ResolvedSymbols = symbols.Values.ToArray (); @@ -99,92 +93,42 @@ public override bool RunTask () return !Log.HasLoggedErrors; } - void SetAssemblyAbiMetadata (string abi, string assetType, ITaskItem assembly, ITaskItem? symbol, bool isDuplicate) + void SetAssemblyAbiMetadata (string abi, ITaskItem assembly, ITaskItem? symbol) { - if (String.IsNullOrEmpty (abi) || (!isDuplicate && String.Compare ("native", assetType, StringComparison.OrdinalIgnoreCase) != 0)) { - return; + if (String.IsNullOrEmpty (abi)) { + throw new ArgumentException ("must not be null or empty", nameof (abi)); } assembly.SetMetadata ("Abi", abi); - if (symbol != null) { - symbol.SetMetadata ("Abi", abi); - } + symbol?.SetMetadata ("Abi", abi); } - void SetAssemblyAbiMetadata (ITaskItem assembly, ITaskItem? symbol, bool isDuplicate) + void SetAssemblyAbiMetadata (ITaskItem assembly, ITaskItem? symbol) { - string assetType = assembly.GetMetadata ("AssetType"); string rid = assembly.GetMetadata ("RuntimeIdentifier"); - if (!String.IsNullOrEmpty (assembly.GetMetadata ("Culture")) || String.Compare ("resources", assetType, StringComparison.OrdinalIgnoreCase) == 0) { - // Satellite assemblies are abi-agnostic, they shouldn't have the Abi metadata set - return; - } - SetAssemblyAbiMetadata (AndroidRidAbiHelper.RuntimeIdentifierToAbi (rid), assetType, assembly, symbol, isDuplicate); + SetAssemblyAbiMetadata (AndroidRidAbiHelper.RuntimeIdentifierToAbi (rid), assembly, symbol); } void SetMetadataForAssemblies (List output, Dictionary symbols) { - foreach (var assembly in InputAssemblies) { - var symbol = GetOrCreateSymbolItem (symbols, assembly); - SetAssemblyAbiMetadata (assembly, symbol, isDuplicate: false); - symbol?.SetDestinationSubPath (); - assembly.SetDestinationSubPath (); + foreach (ITaskItem assembly in InputAssemblies) { + if (DesignTimeBuild && !File.Exists (assembly.ItemSpec)) { + // Designer builds don't produce assemblies, so library and main application DLLs might not + // be there and would later cause an error when the `_CopyAssembliesForDesigner` task runs + continue; + } + + ITaskItem? symbol = GetOrCreateSymbolItem (symbols, assembly); + SetAssemblyAbiMetadata (assembly, symbol); + SetDestinationSubDirectory (assembly, symbol); assembly.SetMetadata ("FrameworkAssembly", IsFromAKnownRuntimePack (assembly).ToString ()); - assembly.SetMetadata ("HasMonoAndroidReference", MonoAndroidHelper.HasMonoAndroidReference (assembly).ToString ()); - output.Add (assembly); - } - } - void DeduplicateAssemblies (List output, Dictionary symbols) - { - // Group by assembly file name - foreach (var group in InputAssemblies.Where (Filter).GroupBy (a => Path.GetFileName (a.ItemSpec))) { - // Get the unique list of MVIDs - var mvids = new HashSet (); - bool? frameworkAssembly = null, hasMonoAndroidReference = null; - foreach (var assembly in group) { - using var pe = new PEReader (File.OpenRead (assembly.ItemSpec)); - var reader = pe.GetMetadataReader (); - var module = reader.GetModuleDefinition (); - var mvid = reader.GetGuid (module.Mvid); - mvids.Add (mvid); - - // Calculate %(FrameworkAssembly) and %(HasMonoAndroidReference) for the first - if (frameworkAssembly == null) { - frameworkAssembly = IsFromAKnownRuntimePack (assembly); - } - if (hasMonoAndroidReference == null) { - hasMonoAndroidReference = MonoAndroidHelper.IsMonoAndroidAssembly (assembly) || - MonoAndroidHelper.HasMonoAndroidReference (reader); - } - assembly.SetMetadata ("FrameworkAssembly", frameworkAssembly.ToString ()); - assembly.SetMetadata ("HasMonoAndroidReference", hasMonoAndroidReference.ToString ()); - } - // If we end up with more than 1 unique mvid, we need *all* assemblies - if (mvids.Count > 1) { - foreach (var assembly in group) { - var symbol = GetOrCreateSymbolItem (symbols, assembly); - SetDestinationSubDirectory (assembly, group.Key, symbol, isDuplicate: true); - output.Add (assembly); - } - } else { - // Otherwise only include the first assembly - bool first = true; - foreach (var assembly in group) { - if (first) { - first = false; - - var symbol = GetOrCreateSymbolItem (symbols, assembly); - symbol?.SetDestinationSubPath (); - assembly.SetDestinationSubPath (); - output.Add (assembly); - SetAssemblyAbiMetadata (assembly, symbol, false); - } else { - symbols.Remove (Path.ChangeExtension (assembly.ItemSpec, ".pdb")); - } - } + if (!DesignTimeBuild) { + // Designer builds don't produce assemblies, the HasMonoAndroidReference call would throw an exception in that case + assembly.SetMetadata ("HasMonoAndroidReference", MonoAndroidHelper.HasMonoAndroidReference (assembly).ToString ()); } + output.Add (assembly); } } @@ -219,35 +163,30 @@ bool Filter (ITaskItem item) /// /// Sets %(DestinationSubDirectory) and %(DestinationSubPath) based on %(RuntimeIdentifier) /// - void SetDestinationSubDirectory (ITaskItem assembly, string fileName, ITaskItem? symbol, bool isDuplicate) + void SetDestinationSubDirectory (ITaskItem assembly, ITaskItem? symbol) { - var rid = assembly.GetMetadata ("RuntimeIdentifier"); - string assetType = assembly.GetMetadata ("AssetType"); - - // Satellite assemblies have `RuntimeIdentifier` set, but they shouldn't - they aren't specific to any architecture, so they should have none of the - // abi-specific metadata set - // - if (!String.IsNullOrEmpty (assembly.GetMetadata ("Culture")) || - String.Compare ("resources", assetType, StringComparison.OrdinalIgnoreCase) == 0) { - rid = String.Empty; + string? rid = assembly.GetMetadata ("RuntimeIdentifier"); + if (String.IsNullOrEmpty (rid)) { + throw new InvalidOperationException ($"Assembly '{assembly}' item is missing required "); } - var abi = AndroidRidAbiHelper.RuntimeIdentifierToAbi (rid); - if (!string.IsNullOrEmpty (abi)) { - string destination = Path.Combine (assembly.GetMetadata ("DestinationSubDirectory"), abi); - assembly.SetMetadata ("DestinationSubDirectory", destination + Path.DirectorySeparatorChar); - assembly.SetMetadata ("DestinationSubPath", Path.Combine (destination, fileName)); - if (symbol != null) { - destination = Path.Combine (symbol.GetMetadata ("DestinationSubDirectory"), abi); - symbol.SetMetadata ("DestinationSubDirectory", destination + Path.DirectorySeparatorChar); - symbol.SetMetadata ("DestinationSubPath", Path.Combine (destination, Path.GetFileName (symbol.ItemSpec))); + string? abi = AndroidRidAbiHelper.RuntimeIdentifierToAbi (rid); + if (string.IsNullOrEmpty (abi)) { + throw new InvalidOperationException ($"Unable to convert a runtime identifier '{rid}' to Android ABI for: {assembly.ItemSpec}"); + } + + SetIt (assembly); + SetIt (symbol); + + void SetIt (ITaskItem? item) + { + if (item == null) { + return; } - SetAssemblyAbiMetadata (abi, assetType, assembly, symbol, isDuplicate); - } else { - Log.LogDebugMessage ($"Android ABI not found for: {assembly.ItemSpec}"); - assembly.SetDestinationSubPath (); - symbol?.SetDestinationSubPath (); + string destination = Path.Combine (abi, item.GetMetadata ("DestinationSubDirectory")); + item.SetMetadata ("DestinationSubDirectory", destination + Path.DirectorySeparatorChar); + item.SetMetadata ("DestinationSubPath", Path.Combine (destination, Path.GetFileName (item.ItemSpec))); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs index e1884c1c5f3..bf69140c390 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs @@ -173,7 +173,7 @@ public void BuildAotApplicationWithNdkAndBundleAndÜmläüts (string supportedAb }; proj.SetProperty ("AndroidNdkDirectory", AndroidNdkPath); - proj.SetAndroidSupportedAbis (supportedAbis); + proj.SetRuntimeIdentifiers (supportedAbis.Split (';')); proj.SetProperty ("EnableLLVM", enableLLVM.ToString ()); proj.SetProperty ("AndroidUseAssemblyStore", usesAssemblyBlobs.ToString ()); bool checkMinLlvmPath = enableLLVM && (supportedAbis == "armeabi-v7a" || supportedAbis == "x86"); @@ -197,7 +197,7 @@ public void BuildAotApplicationWithNdkAndBundleAndÜmläüts (string supportedAb proj.OutputPath, $"{proj.PackageName}-Signed.apk"); var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs); - Assert.IsTrue (helper.Exists ("assemblies/UnnamedProject.dll"), $"UnnamedProject.dll should be in the {proj.PackageName}-Signed.apk"); + Assert.IsTrue (helper.Exists ($"assemblies/{abi}/UnnamedProject.dll"), $"{abi}/UnnamedProject.dll should be in {proj.PackageName}-Signed.apk"); using (var zipFile = ZipHelper.OpenZip (apk)) { Assert.IsNotNull (ZipHelper.ReadFileFromZip (zipFile, string.Format ("lib/{0}/libaot-UnnamedProject.dll.so", abi)), @@ -228,7 +228,7 @@ public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableL AotAssemblies = true, PackageName = "com.xamarin.buildaotappandbundlewithspecialchars", }; - proj.SetAndroidSupportedAbis (supportedAbis); + proj.SetRuntimeIdentifiers (supportedAbis.Split (';')); proj.SetProperty ("EnableLLVM", enableLLVM.ToString ()); proj.SetProperty ("AndroidUseAssemblyStore", usesAssemblyBlobs.ToString ()); using (var b = CreateApkBuilder (path)) { @@ -242,7 +242,7 @@ public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableL proj.OutputPath, $"{proj.PackageName}-Signed.apk"); var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs); - Assert.IsTrue (helper.Exists ("assemblies/UnnamedProject.dll"), $"UnnamedProject.dll should be in the {proj.PackageName}-Signed.apk"); + Assert.IsTrue (helper.Exists ($"assemblies/{abi}/UnnamedProject.dll"), $"{abi}/UnnamedProject.dll should be in {proj.PackageName}-Signed.apk"); using (var zipFile = ZipHelper.OpenZip (apk)) { Assert.IsNotNull (ZipHelper.ReadFileFromZip (zipFile, string.Format ("lib/{0}/libaot-UnnamedProject.dll.so", abi)), diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs index f562bd6e5c3..b0b69ff759b 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs @@ -265,7 +265,14 @@ public void CheckAssemblyCounts (bool isRelease, bool aot) string apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); var helper = new ArchiveAssemblyHelper (apk, useAssemblyStores: true); - Assert.IsTrue (app_config.number_of_assemblies_in_apk == (uint)helper.GetNumberOfAssemblies (), "Assembly count must be equal between ApplicationConfig and the archive contents"); + foreach (string abi in abis) { + AndroidTargetArch arch = MonoAndroidHelper.AbiToTargetArch (abi); + Assert.AreEqual ( + app_config.number_of_assemblies_in_apk, + helper.GetNumberOfAssemblies (arch: arch), + $"Assembly count must be equal between ApplicationConfig and the archive contents for architecture {arch} (ABI: {abi})" + ); + } } } @@ -433,21 +440,6 @@ public void ApplicationIdPlaceholder () } } - [Test] - [Category ("XamarinBuildDownload")] - public void ExtraAaptManifest () - { - var proj = new XamarinAndroidApplicationProject (); - proj.MainActivity = proj.DefaultMainActivity.Replace ("base.OnCreate (bundle);", "base.OnCreate (bundle);\nFirebase.Crashlytics.FirebaseCrashlytics.Instance.SendUnsentReports();"); - proj.PackageReferences.Add (new Package { Id = "Xamarin.Firebase.Crashlytics", Version = "118.5.1.1" }); - proj.PackageReferences.Add (KnownPackages.Xamarin_Build_Download); - using var builder = CreateApkBuilder (); - Assert.IsTrue (builder.Build (proj), "Build should have succeeded."); - var manifest = File.ReadAllText (Path.Combine (Root, builder.ProjectDirectory, "obj", "Debug", "android", "AndroidManifest.xml")); - Assert.IsTrue (manifest.Contains ($"android:authorities=\"{proj.PackageName}.firebaseinitprovider\""), "placeholder not replaced"); - Assert.IsFalse (manifest.Contains ("dollar_openBracket_applicationId_closeBracket"), "`aapt/AndroidManifest.xml` not ignored"); - } - [Test] public void AarContentExtraction () { @@ -786,7 +778,7 @@ public void IfAndroidJarDoesNotExistThrowXA5207 ([Values(true, false)] bool buil Assert.IsTrue (builder.LastBuildOutput.ContainsText ($"Could not find android.jar for API level {proj.TargetSdkVersion}"), "XA5207 should have had a good error message."); if (buildingInsideVisualStudio) Assert.IsTrue (builder.LastBuildOutput.ContainsText ($"Either install it in the Android SDK Manager"), "XA5207 should have an error message for Visual Studio."); - else + else Assert.IsTrue (builder.LastBuildOutput.ContainsText ($"You can install the missing API level by running"), "XA5207 should have an error message for the command line."); } Directory.Delete (AndroidSdkDirectory, recursive: true); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs index e3050b3fb88..507fb5c8aef 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs @@ -42,11 +42,15 @@ public partial class BuildTest2 : BaseTest public void MarshalMethodsDefaultEnabledStatus (bool isRelease, bool marshalMethodsEnabled) { var abis = new [] { "armeabi-v7a", "x86" }; + AndroidTargetArch[] supportedArches = new [] { + AndroidTargetArch.Arm, + AndroidTargetArch.X86, + }; var proj = new XamarinAndroidApplicationProject { IsRelease = isRelease }; proj.SetProperty (KnownProperties.AndroidEnableMarshalMethods, marshalMethodsEnabled.ToString ()); - proj.SetAndroidSupportedAbis (abis); + proj.SetRuntimeIdentifiers (abis); bool shouldMarshalMethodsBeEnabled = isRelease && marshalMethodsEnabled; using (var b = CreateApkBuilder ()) { @@ -57,7 +61,11 @@ public void MarshalMethodsDefaultEnabledStatus (bool isRelease, bool marshalMeth ); string objPath = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath); - List envFiles = EnvironmentHelper.GatherEnvironmentFiles (objPath, String.Join (";", abis), true); + List envFiles = EnvironmentHelper.GatherEnvironmentFiles ( + objPath, + String.Join (";", supportedArches.Select (arch => MonoAndroidHelper.ArchToAbi (arch))), + true + ); EnvironmentHelper.ApplicationConfig app_config = EnvironmentHelper.ReadApplicationConfig (envFiles); Assert.That (app_config, Is.Not.Null, "application_config must be present in the environment files"); @@ -1172,8 +1180,12 @@ public void BuildBasicApplicationCheckPdb () var proj = new XamarinAndroidApplicationProject (); using (var b = CreateApkBuilder ()) { Assert.IsTrue (b.Build (proj), "Build should have succeeded."); - Assert.IsTrue (File.Exists (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, "android/assets/UnnamedProject.pdb")), - "UnnamedProject.pdb must be copied to the Intermediate directory"); + foreach (string rid in b.GetBuildRuntimeIdentifiers ()) { + string abi = MonoAndroidHelper.RidToAbi (rid); + + Assert.IsTrue (File.Exists (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, $"android/assets/{abi}/UnnamedProject.pdb")), + $"UnnamedProject.pdb must be copied to the Intermediate directory for ABI {abi}"); + } } } @@ -1183,11 +1195,22 @@ public void BuildBasicApplicationCheckPdbRepeatBuild () var proj = new XamarinAndroidApplicationProject (); using (var b = CreateApkBuilder ()) { Assert.IsTrue (b.Build (proj), "Build should have succeeded."); - Assert.IsTrue (File.Exists (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, "android/assets/UnnamedProject.pdb")), - "UnnamedProject.pdb must be copied to the Intermediate directory"); + + foreach (string rid in b.GetBuildRuntimeIdentifiers ()) { + string abi = MonoAndroidHelper.RidToAbi (rid); + + Assert.IsTrue (File.Exists (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, $"android/assets/{abi}/UnnamedProject.pdb")), + $"UnnamedProject.pdb must be copied to the Intermediate directory for ABI {abi}"); + } + Assert.IsTrue (b.Build (proj), "second build failed"); - Assert.IsTrue (File.Exists (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, "android/assets/UnnamedProject.pdb")), - "UnnamedProject.pdb must be copied to the Intermediate directory"); + + foreach (string rid in b.GetBuildRuntimeIdentifiers ()) { + string abi = MonoAndroidHelper.RidToAbi (rid); + + Assert.IsTrue (File.Exists (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, $"android/assets/{abi}/UnnamedProject.pdb")), + $"UnnamedProject.pdb must be copied to the Intermediate directory for ABI {abi}"); + } } } @@ -1240,33 +1263,83 @@ public Class2 () Assert.IsTrue (b.Build (proj), "App1 Build should have succeeded."); var intermediate = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath); var outputPath = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath); - var assetsPdb = Path.Combine (intermediate, "android", "assets", "Library1.pdb"); - var binSrc = Path.Combine (outputPath, "Library1.pdb"); - Assert.IsTrue ( - File.Exists (Path.Combine (intermediate, "android", "assets", "Mono.Android.pdb")), - "Mono.Android.pdb must be copied to Intermediate directory"); + + const string LibraryBaseName = "Library1"; + const string AppBaseName = "App1"; + const string LibraryPdbName = LibraryBaseName + ".pdb"; + const string LibraryDllName = LibraryBaseName + ".dll"; + const string AppPdbName = AppBaseName + ".pdb"; + const string AppDllName = AppBaseName + ".dll"; + + string libraryPdbBinSrc = Path.Combine (outputPath, LibraryPdbName); + string appPdbBinSrc = Path.Combine (outputPath, AppPdbName); + Assert.IsTrue ( - File.Exists (assetsPdb), - "Library1.pdb must be copied to Intermediate directory"); + File.Exists (libraryPdbBinSrc), + $"{LibraryPdbName} must be copied to bin directory"); + Assert.IsTrue ( - File.Exists (binSrc), - "Library1.pdb must be copied to bin directory"); - using (var apk = ZipHelper.OpenZip (Path.Combine (outputPath, proj.PackageName + "-Signed.apk"))) { - var data = ZipHelper.ReadFileFromZip (apk, "assemblies/Library1.pdb"); - if (data == null) - data = File.ReadAllBytes (assetsPdb); - var filedata = File.ReadAllBytes (binSrc); - Assert.AreEqual (filedata.Length, data.Length, "Library1.pdb in the apk should match {0}", binSrc); + File.Exists (appPdbBinSrc), + $"{AppPdbName} must be copied to bin directory"); + + var fileNames = new List<(string path, bool existsInBin)> { + ("Mono.Android.pdb", false), + (AppPdbName, true), + (LibraryPdbName, true), + (AppDllName, true), + (LibraryDllName, true), + }; + + string apkPath = Path.Combine (outputPath, proj.PackageName + "-Signed.apk"); + var helper = new ArchiveAssemblyHelper (apkPath, useAssemblyStores: false, b.GetBuildRuntimeIdentifiers ().ToArray ()); + foreach (string abi in b.GetBuildAbis ()) { + foreach ((string fileName, bool existsInBin) in fileNames) { + EnsureFilesAreTheSame (intermediate, existsInBin ? outputPath : null, fileName, abi, helper, uncompressIfNecessary: fileName.EndsWith (".dll", StringComparison.Ordinal)); + } } - var androidAssets = Path.Combine (intermediate, "android", "assets", "App1.pdb"); - binSrc = Path.Combine (outputPath, "App1.pdb"); - Assert.IsTrue ( - File.Exists (binSrc), - "App1.pdb must be copied to bin directory"); - FileAssert.AreEqual (binSrc, androidAssets, "{0} and {1} should not differ.", binSrc, androidAssets); - androidAssets = Path.Combine (intermediate, "android", "assets", "App1.dll"); - binSrc = Path.Combine (outputPath, "App1.dll"); - FileAssert.AreEqual (binSrc, androidAssets, "{0} and {1} should match.", binSrc, androidAssets); + } + } + + void EnsureFilesAreTheSame (string intermediatePath, string? binPath, string fileName, string abi, ArchiveAssemblyHelper helper, bool uncompressIfNecessary) + { + string assetsFilePath = Path.Combine (intermediatePath, "android", "assets", abi, fileName); + Assert.IsTrue (File.Exists (assetsFilePath), $"'{fileName}' must be copied to Intermediate directory for ABI {abi}"); + + using var assetsFileStream = File.OpenRead (assetsFilePath); + string apkEntryPath = MonoAndroidHelper.MakeZipArchivePath ("assemblies", abi, fileName); + using Stream? apkEntryStream = helper.ReadEntry (apkEntryPath, MonoAndroidHelper.AbiToTargetArch (abi), uncompressIfNecessary); + + if (apkEntryStream != null) { // FastDev won't put assemblies in the APK + FileAssert.AreEqual (apkEntryStream, assetsFileStream, $"'{apkEntryPath}' and '{assetsFilePath}' should not differ"); + } + + if (String.IsNullOrEmpty (binPath)) { + return; + } + + // This is a bit fragile. We don't know which RID the `bin/` files were copied from, so we'll do our best to compare + // oranges to oranges by looking at file sizes before attempting the compare. This is a very weak predicate, because + // the files may differ in e.g. the MVID and still have the same size. The real fix for this is to have per-rid `bin/` + // subdirectories. + string binFilePath = Path.Combine (binPath, fileName); + Assert.IsTrue (File.Exists (binFilePath), $"'{fileName}' must be copied to the Output directory"); + + var assetsInfo = new FileInfo (assetsFilePath); + var binInfo = new FileInfo (binFilePath); + + if (assetsInfo.Length != binInfo.Length) { + Assert.Warn ($"Ignoring comparison of '{binFilePath}' with '{assetsFilePath}' because their sizes differ"); + return; + } + + using var binFileStream = File.OpenRead (binFilePath); + assetsFileStream.Seek (0, SeekOrigin.Begin); + FileAssert.AreEqual (assetsFileStream, binFileStream, $"'{assetsFilePath}' and '{binFilePath}' should not differ"); + + if (apkEntryStream != null) { + binFileStream.Seek (0, SeekOrigin.Begin); + apkEntryStream.Seek (0, SeekOrigin.Begin); + FileAssert.AreEqual (apkEntryStream, binFileStream, $"'{apkEntryPath}' and '{binFilePath}' should not differ"); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs index e184f5edce2..b4eae1ec907 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs @@ -430,14 +430,18 @@ public void AppProjectTargetsDoNotBreak () var output = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath); var intermediate = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath); - var filesToTouch = new [] { + var filesToTouch = new List { Path.Combine (intermediate, "..", "project.assets.json"), Path.Combine (intermediate, "build.props"), Path.Combine (intermediate, $"{proj.ProjectName}.dll"), Path.Combine (intermediate, $"{proj.ProjectName}.pdb"), - Path.Combine (intermediate, "android", "assets", $"{proj.ProjectName}.dll"), Path.Combine (output, $"{proj.ProjectName}.dll.config"), }; + + foreach (string abi in b.GetBuildAbis ()) { + filesToTouch.Add (Path.Combine (intermediate, "android", "assets", abi, $"{proj.ProjectName}.dll")); + } + foreach (var file in filesToTouch) { FileAssert.Exists (file); File.SetLastWriteTimeUtc (file, DateTime.UtcNow); @@ -642,8 +646,11 @@ public void TransitiveDependencyProduceReferenceAssembly () Assert.IsTrue (appBuilder.Build (app, doNotCleanupOnUpdate: true, saveProject: false), "app SignAndroidPackage build should have succeeded."); var lib2Output = Path.Combine (path, lib2.ProjectName, "bin", "Debug", "netstandard2.0", $"{lib2.ProjectName}.dll"); - var lib2InAppOutput = Path.Combine (path, app.ProjectName, app.IntermediateOutputPath, "android", "assets", $"{lib2.ProjectName}.dll"); - FileAssert.AreEqual (lib2Output, lib2InAppOutput, "new Library2 should have been copied to app output directory"); + + foreach (string abi in appBuilder.GetBuildAbis ()) { + var lib2InAppOutput = Path.Combine (path, app.ProjectName, app.IntermediateOutputPath, "android", "assets", abi, $"{lib2.ProjectName}.dll"); + FileAssert.AreEqual (lib2Output, lib2InAppOutput, $"new Library2 should have been copied to app output directory for abi '{abi}'"); + } } } @@ -655,8 +662,11 @@ public void LinkAssembliesNoShrink () Assert.IsTrue (b.Build (proj), "build should have succeeded."); // Touch an assembly to a timestamp older than build.props - var formsViewGroup = b.Output.GetIntermediaryPath (Path.Combine ("android", "assets", "FormsViewGroup.dll")); - File.SetLastWriteTimeUtc (formsViewGroup, new DateTime (1970, 1, 1)); + foreach (string rid in b.GetBuildRuntimeIdentifiers ()) { + string abi = MonoAndroidHelper.RidToAbi (rid); + var formsViewGroup = b.Output.GetIntermediaryPath (Path.Combine ("android", "assets", abi, "FormsViewGroup.dll")); + File.SetLastWriteTimeUtc (formsViewGroup, new DateTime (1970, 1, 1)); + } Assert.IsTrue (b.Build (proj, doNotCleanupOnUpdate: true), "build should have succeeded."); b.Output.AssertTargetIsNotSkipped (KnownTargets.LinkAssembliesNoShrink); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs index 122500d6428..35523031a2d 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs @@ -39,8 +39,13 @@ public void CheckIncludedAssemblies ([Values (false, true)] bool usesAssemblySto var proj = new XamarinAndroidApplicationProject { IsRelease = true }; + + AndroidTargetArch[] supportedArches = new[] { + AndroidTargetArch.Arm, + }; + proj.SetProperty ("AndroidUseAssemblyStore", usesAssemblyStores.ToString ()); - proj.SetAndroidSupportedAbis ("armeabi-v7a"); + proj.SetRuntimeIdentifiers (supportedArches); proj.PackageReferences.Add (new Package { Id = "Humanizer.Core", Version = "2.14.1", @@ -59,23 +64,23 @@ public void CheckIncludedAssemblies ([Values (false, true)] bool usesAssemblySto proj.OtherBuildItems.Add (new BuildItem ("Using", "System.Globalization")); proj.OtherBuildItems.Add (new BuildItem ("Using", "Humanizer")); - var expectedFiles = new [] { - "Java.Interop.dll", - "Mono.Android.dll", - "Mono.Android.Runtime.dll", - "rc.bin", - "System.Console.dll", - "System.Private.CoreLib.dll", - "System.Runtime.dll", - "System.Runtime.InteropServices.dll", - "System.Linq.dll", - "UnnamedProject.dll", - "_Microsoft.Android.Resource.Designer.dll", - "Humanizer.dll", - "es/Humanizer.resources.dll", - "System.Collections.dll", - "System.Collections.Concurrent.dll", - "System.Text.RegularExpressions.dll", + var expectedFiles = new HashSet { + "Java.Interop.dll", + "Mono.Android.dll", + "Mono.Android.Runtime.dll", + "System.Console.dll", + "System.Private.CoreLib.dll", + "System.Runtime.dll", + "System.Runtime.InteropServices.dll", + "System.Linq.dll", + "UnnamedProject.dll", + "_Microsoft.Android.Resource.Designer.dll", + "Humanizer.dll", + "es/Humanizer.resources.dll", + "System.Collections.dll", + "System.Collections.Concurrent.dll", + "System.Text.RegularExpressions.dll", + "libarc.bin.so", }; using (var b = CreateApkBuilder ()) { @@ -87,15 +92,11 @@ public void CheckIncludedAssemblies ([Values (false, true)] bool usesAssemblySto List missingFiles; List additionalFiles; - helper.Contains (expectedFiles, out existingFiles, out missingFiles, out additionalFiles); + helper.Contains (expectedFiles, out existingFiles, out missingFiles, out additionalFiles, supportedArches); Assert.IsTrue (missingFiles == null || missingFiles.Count == 0, string.Format ("The following Expected files are missing. {0}", string.Join (Environment.NewLine, missingFiles))); - - Assert.IsTrue (additionalFiles == null || additionalFiles.Count == 0, - string.Format ("Unexpected Files found! {0}", - string.Join (Environment.NewLine, additionalFiles))); } } @@ -465,7 +466,10 @@ public void MissingSatelliteAssemblyInLibrary () var helper = new ArchiveAssemblyHelper (apk); foreach (string lang in languages) { - Assert.IsTrue (helper.Exists ($"assemblies/{lang}/{lib.ProjectName}.resources.dll"), $"Apk should contain satellite assembly for language '{lang}'!"); + foreach (string rid in appBuilder.GetBuildRuntimeIdentifiers ()) { + string abi = MonoAndroidHelper.RidToAbi (rid); + Assert.IsTrue (helper.Exists ($"assemblies/{abi}/{lang}/{lib.ProjectName}.resources.dll"), $"Apk should contain satellite assembly for language '{lang}'!"); + } } } } @@ -494,7 +498,10 @@ public void MissingSatelliteAssemblyInApp () var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); var helper = new ArchiveAssemblyHelper (apk); - Assert.IsTrue (helper.Exists ($"assemblies/es/{proj.ProjectName}.resources.dll"), "Apk should contain satellite assemblies!"); + foreach (string rid in b.GetBuildRuntimeIdentifiers ()) { + string abi = MonoAndroidHelper.RidToAbi (rid); + Assert.IsTrue (helper.Exists ($"assemblies/{abi}/es/{proj.ProjectName}.resources.dll"), "Apk should contain satellite assemblies!"); + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/SingleProjectTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/SingleProjectTest.cs index aabd090f01e..cce32043203 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/SingleProjectTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/SingleProjectTest.cs @@ -84,22 +84,25 @@ public void AndroidManifestProperties (string versionName, string versionCode, s var versionNumber = index == -1 ? $"{versionName}.0.0" : $"{versionName.Substring (0, index)}.0.0"; - var assemblyPath = b.Output.GetIntermediaryPath ($"android/assets/{proj.ProjectName}.dll"); - FileAssert.Exists (assemblyPath); - using var assembly = AssemblyDefinition.ReadAssembly (assemblyPath); - - // System.Reflection.AssemblyVersion - Assert.AreEqual (versionNumber, assembly.Name.Version.ToString ()); - - // System.Reflection.AssemblyFileVersion - var assemblyInfoVersion = assembly.CustomAttributes.FirstOrDefault (a => a.AttributeType.FullName == "System.Reflection.AssemblyInformationalVersionAttribute"); - Assert.IsNotNull (assemblyInfoVersion, "Should find AssemblyInformationalVersionAttribute!"); - Assert.AreEqual (versionName, assemblyInfoVersion.ConstructorArguments [0].Value); - - // System.Reflection.AssemblyInformationalVersion - var assemblyFileVersion = assembly.CustomAttributes.FirstOrDefault (a => a.AttributeType.FullName == "System.Reflection.AssemblyFileVersionAttribute"); - Assert.IsNotNull (assemblyFileVersion, "Should find AssemblyFileVersionAttribute!"); - Assert.AreEqual (versionNumber, assemblyFileVersion.ConstructorArguments [0].Value); + + foreach (string abi in b.GetBuildAbis ()) { + var assemblyPath = b.Output.GetIntermediaryPath ($"android/assets/{abi}/{proj.ProjectName}.dll"); + FileAssert.Exists (assemblyPath); + using var assembly = AssemblyDefinition.ReadAssembly (assemblyPath); + + // System.Reflection.AssemblyVersion + Assert.AreEqual (versionNumber, assembly.Name.Version.ToString ()); + + // System.Reflection.AssemblyFileVersion + var assemblyInfoVersion = assembly.CustomAttributes.FirstOrDefault (a => a.AttributeType.FullName == "System.Reflection.AssemblyInformationalVersionAttribute"); + Assert.IsNotNull (assemblyInfoVersion, "Should find AssemblyInformationalVersionAttribute!"); + Assert.AreEqual (versionName, assemblyInfoVersion.ConstructorArguments [0].Value); + + // System.Reflection.AssemblyInformationalVersion + var assemblyFileVersion = assembly.CustomAttributes.FirstOrDefault (a => a.AttributeType.FullName == "System.Reflection.AssemblyFileVersionAttribute"); + Assert.IsNotNull (assemblyFileVersion, "Should find AssemblyFileVersionAttribute!"); + Assert.AreEqual (versionNumber, assemblyFileVersion.ConstructorArguments [0].Value); + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/GeneratePackageManagerJavaTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/GeneratePackageManagerJavaTests.cs index 4f75c742ab2..e61f0cec2f3 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/GeneratePackageManagerJavaTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/GeneratePackageManagerJavaTests.cs @@ -1,4 +1,5 @@ using NUnit.Framework; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -61,8 +62,12 @@ public void CheckPackageManagerAssemblyOrder (string[] resolvedUserAssemblies, s File.WriteAllText (Path.Combine (path, "AndroidManifest.xml"), $@""); - var resolvedUserAssembliesList = resolvedUserAssemblies.Select (x => new TaskItem (x)); - var resolvedAssembliesList = resolvedAssemblies.Select (x => new TaskItem (x)); + var metadata = new Dictionary (StringComparer.OrdinalIgnoreCase) { + {"Abi", "arm64-v8a"}, + }; + + var resolvedUserAssembliesList = resolvedUserAssemblies.Select (x => new TaskItem (x, metadata)); + var resolvedAssembliesList = resolvedAssemblies.Select (x => new TaskItem (x, metadata)); var task = new GeneratePackageManagerJava { BuildEngine = new MockBuildEngine (TestContext.Out), diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs index 76337da66aa..86f2a319cc0 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs @@ -9,6 +9,7 @@ using Mono.Tuner; using MonoDroid.Tuner; using NUnit.Framework; +using Xamarin.Android.Tasks; using Xamarin.ProjectTools; using SR = System.Reflection; @@ -228,26 +229,29 @@ public void RemoveDesigner ([Values (true, false)] bool useAssemblyStore) proj.SetProperty ("AndroidLinkResources", "True"); proj.SetProperty ("AndroidUseAssemblyStore", useAssemblyStore.ToString ()); string assemblyName = proj.ProjectName; - using (var b = CreateApkBuilder ()) { - Assert.IsTrue (b.Build (proj), "build should have succeeded."); - var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); - FileAssert.Exists (apk); - var helper = new ArchiveAssemblyHelper (apk, useAssemblyStore); - Assert.IsTrue (helper.Exists ($"assemblies/{assemblyName}.dll"), $"{assemblyName}.dll should exist in apk!"); - using (var stream = helper.ReadEntry ($"assemblies/{assemblyName}.dll")) { - stream.Position = 0; - using (var assembly = AssemblyDefinition.ReadAssembly (stream)) { - var type = assembly.MainModule.GetType ($"{assemblyName}.Resource"); - var intType = typeof(int); - foreach (var nestedType in type.NestedTypes) { - int count = 0; - foreach (var field in nestedType.Fields) { - if (field.FieldType.FullName == intType.FullName) - count++; - } - Assert.AreEqual (0, count, "All Nested Resource Type int fields should be removed."); - } + + using var b = CreateApkBuilder (); + Assert.IsTrue (b.Build (proj), "build should have succeeded."); + var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); + FileAssert.Exists (apk); + var helper = new ArchiveAssemblyHelper (apk, useAssemblyStore); + foreach (string rid in b.GetBuildRuntimeIdentifiers ()) { + string abi = MonoAndroidHelper.RidToAbi (rid); + Assert.IsTrue (helper.Exists ($"assemblies/{abi}/{assemblyName}.dll"), $"{assemblyName}.dll should exist in apk!"); + + using var stream = helper.ReadEntry ($"assemblies/{assemblyName}.dll"); + stream.Position = 0; + + using var assembly = AssemblyDefinition.ReadAssembly (stream); + var type = assembly.MainModule.GetType ($"{assemblyName}.Resource"); + var intType = typeof(int); + foreach (var nestedType in type.NestedTypes) { + int count = 0; + foreach (var field in nestedType.Fields) { + if (field.FieldType.FullName == intType.FullName) + count++; } + Assert.AreEqual (0, count, "All Nested Resource Type int fields should be removed."); } } } @@ -288,7 +292,10 @@ public void LinkDescription ([Values (true, false)] bool useAssemblyStore) var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); FileAssert.Exists (apk); var helper = new ArchiveAssemblyHelper (apk, useAssemblyStore); - Assert.IsTrue (helper.Exists ($"assemblies/{assembly_name}.dll"), $"{assembly_name}.dll should exist in apk!"); + foreach (string rid in b.GetBuildRuntimeIdentifiers ()) { + string abi = MonoAndroidHelper.RidToAbi (rid); + Assert.IsTrue (helper.Exists ($"assemblies/{abi}/{assembly_name}.dll"), $"{assembly_name}.dll should exist in apk!"); + } using (var stream = helper.ReadEntry ($"assemblies/{assembly_name}.dll")) { stream.Position = 0; using (var assembly = AssemblyDefinition.ReadAssembly (stream)) { @@ -422,34 +429,47 @@ public unsafe bool MyMethod (Android.OS.IBinder windowToken, [global::Android.Ru using (var b = CreateApkBuilder ()) { Assert.IsTrue (b.Build (proj), "Building a project should have succeded."); + string projectDir = Path.Combine (proj.Root, b.ProjectDirectory); var assemblyFile = "UnnamedProject.dll"; - var assemblyPath = (!isRelease || setLinkModeNone) ? b.Output.GetIntermediaryPath (Path.Combine ("android", "assets", assemblyFile)) : BuildTest.GetLinkedPath (b, true, assemblyFile); - using (var assembly = AssemblyDefinition.ReadAssembly (assemblyPath)) { - Assert.IsTrue (assembly != null); + if (!isRelease || setLinkModeNone) { + foreach (string abi in b.GetBuildAbis ()) { + CheckAssembly (b.Output.GetIntermediaryPath (Path.Combine ("android", "assets", abi, assemblyFile)), projectDir); + } + } else { + CheckAssembly (BuildTest.GetLinkedPath (b, true, assemblyFile), projectDir); + } + } - var td = assembly.MainModule.GetType ("UnnamedProject.MyClass"); - Assert.IsTrue (td != null); + void CheckAssembly (string assemblyPath, string projectDir) + { + string shortAssemblyPath = Path.GetRelativePath (projectDir, assemblyPath); + Console.WriteLine ($"CheckAssembly for '{shortAssemblyPath}'"); + using var assembly = AssemblyDefinition.ReadAssembly (assemblyPath); + Assert.IsTrue (assembly != null, $"Assembly '${shortAssemblyPath}' should have been loaded"); - var mr = td.GetMethods ().Where (m => m.Name == "MyMethod").FirstOrDefault (); - Assert.IsTrue (mr != null); + var td = assembly.MainModule.GetType ("UnnamedProject.MyClass"); + Assert.IsTrue (td != null, $"`UnnamedProject.MyClass` type definition should have been found in assembly '{shortAssemblyPath}'"); - var md = mr.Resolve (); - Assert.IsTrue (md != null); + var mr = td.GetMethods ().Where (m => m.Name == "MyMethod").FirstOrDefault (); + Assert.IsTrue (mr != null, $"`MyMethod` method reference should have been found (assembly '{shortAssemblyPath}')"); - bool hasKeepAliveCall = false; - foreach (var i in md.Body.Instructions) { - if (i.OpCode.Code != Mono.Cecil.Cil.Code.Call) - continue; + var md = mr.Resolve (); + Assert.IsTrue (md != null, $"`MyMethod` method reference should have been resolved (assembly '{shortAssemblyPath}')"); - if (!i.Operand.ToString ().Contains ("System.GC::KeepAlive")) - continue; + bool hasKeepAliveCall = false; + foreach (var i in md.Body.Instructions) { + if (i.OpCode.Code != Mono.Cecil.Cil.Code.Call) + continue; - hasKeepAliveCall = true; - break; - } + if (!i.Operand.ToString ().Contains ("System.GC::KeepAlive")) + continue; - Assert.IsTrue (hasKeepAliveCall == shouldAddKeepAlives); + hasKeepAliveCall = true; + break; } + + string not = shouldAddKeepAlives ? String.Empty : " not"; + Assert.IsTrue (hasKeepAliveCall == shouldAddKeepAlives, $"KeepAlive call should{not} have been found (assembly '{shortAssemblyPath}')"); } } @@ -599,8 +619,8 @@ void Assert64Bit(string rid, bool expected64) /* * IL snippet - * .method private hidebysig specialname rtspecialname static - * void .cctor () cil managed + * .method private hidebysig specialname rtspecialname static + * void .cctor () cil managed * { * // Is64Bits = 4 >= 8; * IL_0000: ldc.i4 4 diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs index 3966c85180a..e4dfa7b26de 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -5,6 +5,8 @@ using System.Linq; using Xamarin.Android.AssemblyStore; +using Xamarin.Android.Tools; +using Xamarin.Android.Tasks; using Xamarin.ProjectTools; using Xamarin.Tools.Zip; @@ -21,13 +23,6 @@ public class ArchiveAssemblyHelper ".pdb", }; - static readonly Dictionary ArchToAbi = new Dictionary (StringComparer.OrdinalIgnoreCase) { - {"x86", "x86"}, - {"x86_64", "x86_64"}, - {"armeabi_v7a", "armeabi-v7a"}, - {"arm64_v8a", "arm64-v8a"}, - }; - static readonly ArrayPool buffers = ArrayPool.Shared; readonly string archivePath; @@ -50,91 +45,94 @@ public ArchiveAssemblyHelper (string archivePath, bool useAssemblyStores = true, string extension = Path.GetExtension (archivePath) ?? String.Empty; if (String.Compare (".aab", extension, StringComparison.OrdinalIgnoreCase) == 0) { - assembliesRootDir = "base/root/assemblies"; + assembliesRootDir = "base/lib/"; } else if (String.Compare (".apk", extension, StringComparison.OrdinalIgnoreCase) == 0) { - assembliesRootDir = "assemblies/"; + assembliesRootDir = "lib/"; } else if (String.Compare (".zip", extension, StringComparison.OrdinalIgnoreCase) == 0) { - assembliesRootDir = "root/assemblies/"; + assembliesRootDir = "lib/"; } else { assembliesRootDir = String.Empty; } } - public Stream ReadEntry (string path) + public Stream? ReadEntry (string path, AndroidTargetArch arch = AndroidTargetArch.None, bool uncompressIfNecessary = false) { + Stream? ret; if (useAssemblyStores) { - return ReadStoreEntry (path); + ret = ReadStoreEntry (path, arch, uncompressIfNecessary); + } else { + ret = ReadZipEntry (path, arch, uncompressIfNecessary); } - return ReadZipEntry (path); + ret?.Seek (0, SeekOrigin.Begin); + return ret; } - Stream ReadZipEntry (string path) + Stream? ReadZipEntry (string path, AndroidTargetArch arch, bool uncompressIfNecessary) { - using (var zip = ZipHelper.OpenZip (archivePath)) { - ZipEntry entry = zip.ReadEntry (path); + List? potentialEntries = TransformArchiveAssemblyPath (path, arch); + if (potentialEntries == null || potentialEntries.Count == 0) { + return null; + } + + using var zip = ZipHelper.OpenZip (archivePath); + foreach (string assemblyPath in potentialEntries) { + if (!zip.ContainsEntry (assemblyPath)) { + continue; + } + + ZipEntry entry = zip.ReadEntry (assemblyPath); var ret = new MemoryStream (); entry.Extract (ret); ret.Flush (); return ret; } + + return null; } - Stream ReadStoreEntry (string path) + Stream? ReadStoreEntry (string path, AndroidTargetArch arch, bool uncompressIfNecessary) { - AssemblyStoreReader storeReader = null; - AssemblyStoreAssembly assembly = null; string name = Path.GetFileNameWithoutExtension (path); - var explorer = new AssemblyStoreExplorer (archivePath); - - foreach (var asm in explorer.Assemblies) { - if (String.Compare (name, asm.Name, StringComparison.Ordinal) != 0) { - continue; - } - assembly = asm; - storeReader = asm.Store; - break; + (IList? explorers, string? errorMessage) = AssemblyStoreExplorer.Open (archivePath); + AssemblyStoreExplorer? explorer = SelectExplorer (explorers, arch); + if (explorer == null) { + Console.WriteLine ($"Failed to read assembly '{name}' from '{archivePath}'. {errorMessage}"); + return null; } - if (storeReader == null) { - Console.WriteLine ($"Store for entry {path} not found, will try a standard Zip read"); - return ReadZipEntry (path); - } + if (arch == AndroidTargetArch.None) { + if (explorer.TargetArch == null) { + throw new InvalidOperationException ($"Internal error: explorer should not have its TargetArch unset"); + } - string storeEntryName; - if (String.IsNullOrEmpty (storeReader.Arch)) { - storeEntryName = $"{assembliesRootDir}assemblies.blob"; - } else { - storeEntryName = $"{assembliesRootDir}assemblies_{storeReader.Arch}.blob"; + arch = (AndroidTargetArch)explorer.TargetArch; } - Stream store = ReadZipEntry (storeEntryName); - if (store == null) { - Console.WriteLine ($"Store zip entry {storeEntryName} does not exist"); + Console.WriteLine ($"Trying to read store entry: {name}"); + IList? assemblies = explorer.Find (name, arch); + if (assemblies == null) { + Console.WriteLine ($"Failed to locate assembly '{name}' in assembly store for architecture '{arch}', in archive '{archivePath}'"); return null; } - store.Seek (assembly.DataOffset, SeekOrigin.Begin); - var ret = new MemoryStream (); - byte[] buffer = buffers.Rent (AssemblyStoreReadBufferSize); - int toRead = (int)assembly.DataSize; - while (toRead > 0) { - int nread = store.Read (buffer, 0, AssemblyStoreReadBufferSize); - if (nread <= 0) { + AssemblyStoreItem? assembly = null; + foreach (AssemblyStoreItem item in assemblies) { + if (arch == AndroidTargetArch.None || item.TargetArch == arch) { + assembly = item; break; } + } - ret.Write (buffer, 0, nread); - toRead -= nread; + if (assembly == null) { + Console.WriteLine ($"Failed to find assembly '{name}' in assembly store for architecture '{arch}', in archive '{archivePath}'"); + return null; } - ret.Flush (); - store.Dispose (); - buffers.Return (buffer); - return ret; + return explorer.ReadImageData (assembly, uncompressIfNecessary); } - public List ListArchiveContents (string storeEntryPrefix = DefaultAssemblyStoreEntryPrefix, bool forceRefresh = false) + public List ListArchiveContents (string storeEntryPrefix = DefaultAssemblyStoreEntryPrefix, bool forceRefresh = false, AndroidTargetArch arch = AndroidTargetArch.None) { if (!forceRefresh && archiveContents != null) { return archiveContents; @@ -158,24 +156,18 @@ public List ListArchiveContents (string storeEntryPrefix = DefaultAssemb } Console.WriteLine ($"Creating AssemblyStoreExplorer for archive '{archivePath}'"); - var explorer = new AssemblyStoreExplorer (archivePath); - Console.WriteLine ($"Explorer found {explorer.Assemblies.Count} assemblies"); - foreach (var asm in explorer.Assemblies) { - string prefix = storeEntryPrefix; - - if (haveMultipleRids && !String.IsNullOrEmpty (asm.Store.Arch)) { - string arch = ArchToAbi[asm.Store.Arch]; - prefix = $"{prefix}{arch}/"; - } + (IList? explorers, string? errorMessage) = AssemblyStoreExplorer.Open (archivePath); - entries.Add ($"{prefix}{asm.Name}.dll"); - if (asm.DebugDataOffset > 0) { - entries.Add ($"{prefix}{asm.Name}.pdb"); + if (arch == AndroidTargetArch.None) { + if (explorers == null || explorers.Count == 0) { + return entries; } - if (asm.ConfigDataOffset > 0) { - entries.Add ($"{prefix}{asm.Name}.dll.config"); + foreach (AssemblyStoreExplorer? explorer in explorers) { + SynthetizeAssemblies (explorer); } + } else { + SynthetizeAssemblies (SelectExplorer (explorers, arch)); } Console.WriteLine ("Archive entries with synthetised assembly storeReader entries:"); @@ -184,120 +176,382 @@ public List ListArchiveContents (string storeEntryPrefix = DefaultAssemb } return entries; + + void SynthetizeAssemblies (AssemblyStoreExplorer? explorer) + { + if (explorer == null) { + return; + } + + Console.WriteLine ($"Explorer for {explorer.TargetArch} found {explorer.AssemblyCount} assemblies"); + foreach (AssemblyStoreItem asm in explorer.Assemblies) { + string prefix = storeEntryPrefix; + string abi = MonoAndroidHelper.ArchToAbi (asm.TargetArch); + prefix = $"{prefix}{abi}/"; + + int cultureIndex = asm.Name.IndexOf ('/'); + string? culture = null; + string name; + + if (cultureIndex > 0) { + culture = asm.Name.Substring (0, cultureIndex); + name = asm.Name.Substring (cultureIndex + 1); + } else { + name = asm.Name; + } + + // Mangle name in in the same fashion the discrete assembly entries are named, makes other + // code in this class simpler. + string mangledName = MonoAndroidHelper.MakeDiscreteAssembliesEntryName (name, culture); + entries.Add ($"{prefix}{mangledName}"); + if (asm.DebugOffset > 0) { + mangledName = MonoAndroidHelper.MakeDiscreteAssembliesEntryName (Path.ChangeExtension (name, "pdb")); + entries.Add ($"{prefix}{mangledName}"); + } + + if (asm.ConfigOffset > 0) { + mangledName = MonoAndroidHelper.MakeDiscreteAssembliesEntryName (Path.ChangeExtension (name, "config")); + entries.Add ($"{prefix}{mangledName}"); + } + } + } } - public int GetNumberOfAssemblies (bool countAbiAssembliesOnce = true, bool forceRefresh = false) + AssemblyStoreExplorer? SelectExplorer (IList? explorers, string rid) { - List contents = ListArchiveContents (assembliesRootDir, forceRefresh); - var dlls = contents.Where (x => x.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)); + return SelectExplorer (explorers, MonoAndroidHelper.RidToArch (rid)); + } - if (!countAbiAssembliesOnce) { - return dlls.Count (); + AssemblyStoreExplorer? SelectExplorer (IList? explorers, AndroidTargetArch arch) + { + if (explorers == null || explorers.Count == 0) { + return null; } - var cache = new HashSet (StringComparer.OrdinalIgnoreCase); - return dlls.Where (x => { - string name = Path.GetFileName (x); - if (cache.Contains (name)) { + // If we don't care about target architecture, we check the first store, since all of them will have the same + // assemblies. Otherwise we try to locate the correct store. + if (arch == AndroidTargetArch.None) { + return explorers[0]; + } + + foreach (AssemblyStoreExplorer e in explorers) { + if (e.TargetArch == null || e.TargetArch != arch) { + continue; + } + return e; + } + + + Console.WriteLine ($"Failed to find assembly store for architecture '{arch}' in archive '{archivePath}'"); + return null; + } + + public int GetNumberOfAssemblies (bool forceRefresh = false, AndroidTargetArch arch = AndroidTargetArch.None) + { + List contents = ListArchiveContents (assembliesRootDir, forceRefresh, arch); + + // We must count only .dll.so entries starting with the '-' and '_' characters, as they are the actual managed assemblies. + // Other entries in `lib/{arch}` might be AOT shared libraries, which will also have the .dll.so extension. + var dlls = contents.Where (x => { + string fileName = Path.GetFileName (x); + if (!fileName.EndsWith (".dll.so", StringComparison.OrdinalIgnoreCase)) { return false; } - cache.Add (name); - return true; - }).Count (); + return fileName.StartsWith (MonoAndroidHelper.MANGLED_ASSEMBLY_REGULAR_ASSEMBLY_MARKER, StringComparison.OrdinalIgnoreCase) || + fileName.StartsWith (MonoAndroidHelper.MANGLED_ASSEMBLY_SATELLITE_ASSEMBLY_MARKER, StringComparison.OrdinalIgnoreCase); + }); + + return dlls.Count (); } - public bool Exists (string entryPath, bool forceRefresh = false) + /// + /// Takes "old style" `assemblies/assembly.dll` path and returns (if possible) a set of paths that reflect the new + /// location of `lib/{ARCH}/assembly.dll.so`. A list is returned because, if `arch` is `None`, we'll return all + /// the possible architectural paths. + /// An exception is thrown if we cannot transform the path for some reason. It should **not** be handled. + /// + static List? TransformArchiveAssemblyPath (string path, AndroidTargetArch arch) { - List contents = ListArchiveContents (assembliesRootDir, forceRefresh); - if (contents.Count == 0) { + if (String.IsNullOrEmpty (path)) { + throw new ArgumentException (nameof (path), "must not be null or empty"); + } + + if (!path.StartsWith ("assemblies/", StringComparison.Ordinal)) { + return new List { path }; + } + + string[] parts = path.Split ('/'); + if (parts.Length < 2) { + throw new InvalidOperationException ($"Path '{path}' must consist of at least two segments separated by `/`"); + } + + // We accept: + // assemblies/assembly.dll + // assemblies/{CULTURE}/assembly.dll + // assemblies/{ABI}/assembly.dll + // assemblies/{ABI}/{CULTURE}/assembly.dll + if (parts.Length > 4) { + throw new InvalidOperationException ($"Path '{path}' must not consist of more than 4 segments separated by `/`"); + } + + string? fileName = null; + string? culture = null; + string? abi = null; + + switch (parts.Length) { + // Full satellite assembly path, with abi + case 4: + abi = parts[1]; + culture = parts[2]; + fileName = parts[3]; + break; + + // Assembly path with abi or culture + case 3: + // If the middle part isn't a valid abi, we treat it as a culture name + if (MonoAndroidHelper.IsValidAbi (parts[1])) { + abi = parts[1]; + } else { + culture = parts[1]; + } + fileName = parts[2]; + break; + + // Assembly path without abi or culture + case 2: + fileName = parts[1]; + break; + } + + string fileTypeMarker = MonoAndroidHelper.MANGLED_ASSEMBLY_REGULAR_ASSEMBLY_MARKER; + var abis = new List (); + if (!String.IsNullOrEmpty (abi)) { + abis.Add (abi); + } else if (arch == AndroidTargetArch.None) { + foreach (AndroidTargetArch targetArch in MonoAndroidHelper.SupportedTargetArchitectures) { + abis.Add (MonoAndroidHelper.ArchToAbi (targetArch)); + } + } else { + abis.Add (MonoAndroidHelper.ArchToAbi (arch)); + } + + if (!String.IsNullOrEmpty (culture)) { + // Android doesn't allow us to put satellite assemblies in lib/{CULTURE}/assembly.dll.so, we must instead + // mangle the name. + fileTypeMarker = MonoAndroidHelper.MANGLED_ASSEMBLY_SATELLITE_ASSEMBLY_MARKER; + fileName = $"{culture}-{fileName}"; + } + + var ret = new List (); + var newParts = new List { + String.Empty, // ABI placeholder + $"{fileTypeMarker}{fileName}.so", + }; + + foreach (string a in abis) { + newParts[0] = a; + ret.Add (MonoAndroidHelper.MakeZipArchivePath ("lib", newParts)); + } + + return ret; + } + + static bool ArchiveContains (List archiveContents, string entryPath, AndroidTargetArch arch) + { + if (archiveContents.Count == 0) { + return false; + } + + List? potentialEntries = TransformArchiveAssemblyPath (entryPath, arch); + if (potentialEntries == null || potentialEntries.Count == 0) { return false; } - return contents.Contains (entryPath); + foreach (string wantedEntry in potentialEntries) { + Console.WriteLine ($"Wanted entry: {wantedEntry}"); + foreach (string existingEntry in archiveContents) { + if (String.Compare (existingEntry, wantedEntry, StringComparison.Ordinal) == 0) { + return true; + } + } + } + + return false; } - public void Contains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles) + /// + /// Checks whether exists in the archive or assembly store. The path should use the + /// "old style" `assemblies/{ABI}/assembly.dll` format. + /// + public bool Exists (string entryPath, bool forceRefresh = false, AndroidTargetArch arch = AndroidTargetArch.None) + { + return ArchiveContains (ListArchiveContents (assembliesRootDir, forceRefresh), entryPath, arch); + } + + public void Contains (ICollection fileNames, out List existingFiles, out List missingFiles, out List additionalFiles, IEnumerable? targetArches = null) { if (fileNames == null) { throw new ArgumentNullException (nameof (fileNames)); } - if (fileNames.Length == 0) { + if (fileNames.Count == 0) { throw new ArgumentException ("must not be empty", nameof (fileNames)); } if (useAssemblyStores) { - StoreContains (fileNames, out existingFiles, out missingFiles, out additionalFiles); + StoreContains (fileNames, out existingFiles, out missingFiles, out additionalFiles, targetArches); } else { - ArchiveContains (fileNames, out existingFiles, out missingFiles, out additionalFiles); + ArchiveContains (fileNames, out existingFiles, out missingFiles, out additionalFiles, targetArches); + } + } + + List GetSupportedArches (IEnumerable? runtimeIdentifiers) + { + var rids = new List (); + if (runtimeIdentifiers != null) { + rids.AddRange (runtimeIdentifiers); + } + + if (rids.Count == 0) { + rids.AddRange (MonoAndroidHelper.SupportedTargetArchitectures); + } + + return rids; + } + + void ListFiles (List existingFiles, List missingFiles, List additionalFiles) + { + Console.WriteLine ("Archive contents:"); + ListFiles ("existing files", existingFiles); + ListFiles ("missing files", missingFiles); + ListFiles ("additional files", additionalFiles); + + void ListFiles (string label, List list) + { + Console.WriteLine ($" {label}:"); + if (list.Count == 0) { + Console.WriteLine (" none"); + return; + } + + foreach (string file in list) { + Console.WriteLine ($" {file}"); + } } } - void ArchiveContains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles) + (string prefixAssemblies, string prefixLib) GetArchivePrefixes (string abi) => ($"{MonoAndroidHelper.MakeZipArchivePath (assembliesRootDir, abi)}/", $"lib/{abi}/"); + + void ArchiveContains (ICollection fileNames, out List existingFiles, out List missingFiles, out List additionalFiles, IEnumerable? targetArches = null) { - using (var zip = ZipHelper.OpenZip (archivePath)) { - existingFiles = zip.Where (a => a.FullName.StartsWith (assembliesRootDir, StringComparison.InvariantCultureIgnoreCase)).Select (a => a.FullName).ToList (); - missingFiles = fileNames.Where (x => !zip.ContainsEntry (assembliesRootDir + x)).ToList (); - additionalFiles = existingFiles.Where (x => !fileNames.Contains (x.Replace (assembliesRootDir, string.Empty))).ToList (); + using var zip = ZipHelper.OpenZip (archivePath); + existingFiles = zip.Where (a => a.FullName.StartsWith (assembliesRootDir, StringComparison.InvariantCultureIgnoreCase)).Select (a => a.FullName).ToList (); + existingFiles.AddRange (zip.Where (a => a.FullName.StartsWith ("lib/", StringComparison.OrdinalIgnoreCase)).Select (a => a.FullName)); + + List arches = GetSupportedArches (targetArches); + + missingFiles = new List (); + additionalFiles = new List (); + foreach (AndroidTargetArch arch in arches) { + string abi = MonoAndroidHelper.ArchToAbi (arch); + missingFiles.AddRange (GetMissingFilesForAbi (abi)); + additionalFiles.AddRange (GetAdditionalFilesForAbi (abi, existingFiles)); + } + ListFiles (existingFiles, missingFiles, additionalFiles); + + IEnumerable GetMissingFilesForAbi (string abi) + { + (string prefixAssemblies, string prefixLib) = GetArchivePrefixes (abi); + return fileNames.Where (x => { + string? culture = null; + string fileName = x; + int slashIndex = x.IndexOf ('/'); + if (slashIndex > 0) { + culture = x.Substring (0, slashIndex); + fileName = x.Substring (slashIndex + 1); + } + + return !zip.ContainsEntry (MonoAndroidHelper.MakeZipArchivePath (prefixAssemblies, x)) && + !zip.ContainsEntry (MonoAndroidHelper.MakeZipArchivePath (prefixLib, x)) && + !zip.ContainsEntry (MonoAndroidHelper.MakeZipArchivePath (prefixAssemblies, MonoAndroidHelper.MakeDiscreteAssembliesEntryName (fileName, culture))) && + !zip.ContainsEntry (MonoAndroidHelper.MakeZipArchivePath (prefixLib, MonoAndroidHelper.MakeDiscreteAssembliesEntryName (fileName, culture))); + }); + } + + IEnumerable GetAdditionalFilesForAbi (string abi, List existingFiles) + { + (string prefixAssemblies, string prefixLib) = GetArchivePrefixes (abi); + return existingFiles.Where (x => !fileNames.Contains (x.Replace (prefixAssemblies, string.Empty)) && !fileNames.Contains (x.Replace (prefixLib, String.Empty))); } } - void StoreContains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles) + void StoreContains (ICollection fileNames, out List existingFiles, out List missingFiles, out List additionalFiles, IEnumerable? targetArches = null) { var assemblyNames = fileNames.Where (x => x.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)).ToList (); var configFiles = fileNames.Where (x => x.EndsWith (".config", StringComparison.OrdinalIgnoreCase)).ToList (); - var debugFiles = fileNames.Where (x => x.EndsWith (".pdb", StringComparison.OrdinalIgnoreCase) || x.EndsWith (".mdb", StringComparison.OrdinalIgnoreCase)).ToList (); + var debugFiles = fileNames.Where (x => x.EndsWith (".pdb", StringComparison.OrdinalIgnoreCase)).ToList (); var otherFiles = fileNames.Where (x => !SpecialExtensions.Contains (Path.GetExtension (x))).ToList (); existingFiles = new List (); missingFiles = new List (); additionalFiles = new List (); - if (otherFiles.Count > 0) { - using (var zip = ZipHelper.OpenZip (archivePath)) { + using ZipArchive? zip = ZipHelper.OpenZip (archivePath); + + List arches = GetSupportedArches (targetArches); + (IList? explorers, string? errorMessage) = AssemblyStoreExplorer.Open (archivePath); + + foreach (AndroidTargetArch arch in arches) { + AssemblyStoreExplorer? explorer = SelectExplorer (explorers, arch); + if (explorer == null) { + continue; + } + + if (otherFiles.Count > 0) { + (string prefixAssemblies, string prefixLib) = GetArchivePrefixes (MonoAndroidHelper.ArchToAbi (arch)); + foreach (string file in otherFiles) { - string fullPath = assembliesRootDir + file; + string fullPath = prefixAssemblies + file; + if (zip.ContainsEntry (fullPath)) { + existingFiles.Add (file); + } + + fullPath = prefixLib + file; if (zip.ContainsEntry (fullPath)) { existingFiles.Add (file); } } } - } - - var explorer = new AssemblyStoreExplorer (archivePath, customLogger: (a, s) => { - Console.WriteLine ($"DEBUG! {s}"); - }); - foreach (var f in explorer.AssembliesByName) { - Console.WriteLine ($"DEBUG!\tKey:{f.Key}"); - } + foreach (var f in explorer.AssembliesByName) { + Console.WriteLine ($"DEBUG!\tKey:{f.Key}"); + } - // Assembly stores don't store the assembly extension - var storeAssemblies = explorer.AssembliesByName.Keys.Select (x => $"{x}.dll"); - if (explorer.AssembliesByName.Count != 0) { - existingFiles.AddRange (storeAssemblies); + if (explorer.AssembliesByName.Count != 0) { + existingFiles.AddRange (explorer.AssembliesByName.Keys); - // We need to fake config and debug files since they have no named entries in the storeReader - foreach (string file in configFiles) { - AssemblyStoreAssembly asm = GetStoreAssembly (file); - if (asm == null) { - continue; - } + // We need to fake config and debug files since they have no named entries in the storeReader + foreach (string file in configFiles) { + AssemblyStoreItem asm = GetStoreAssembly (explorer, file); + if (asm == null) { + continue; + } - if (asm.ConfigDataOffset > 0) { - existingFiles.Add (file); + if (asm.ConfigOffset > 0) { + existingFiles.Add (file); + } } - } - foreach (string file in debugFiles) { - AssemblyStoreAssembly asm = GetStoreAssembly (file); - if (asm == null) { - continue; - } + foreach (string file in debugFiles) { + AssemblyStoreItem asm = GetStoreAssembly (explorer, file); + if (asm == null) { + continue; + } - if (asm.DebugDataOffset > 0) { - existingFiles.Add (file); + if (asm.DebugOffset > 0) { + existingFiles.Add (file); + } } } } @@ -310,15 +564,12 @@ void StoreContains (string[] fileNames, out List existingFiles, out List } additionalFiles = existingFiles.Where (x => !fileNames.Contains (x)).ToList (); + ListFiles (existingFiles, missingFiles, additionalFiles); - AssemblyStoreAssembly GetStoreAssembly (string file) + AssemblyStoreItem GetStoreAssembly (AssemblyStoreExplorer explorer, string file) { string assemblyName = Path.GetFileNameWithoutExtension (file); - if (assemblyName.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) { - assemblyName = Path.GetFileNameWithoutExtension (assemblyName); - } - - if (!explorer.AssembliesByName.TryGetValue (assemblyName, out AssemblyStoreAssembly asm) || asm == null) { + if (!explorer.AssembliesByName.TryGetValue (assemblyName, out AssemblyStoreItem asm) || asm == null) { return null; } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/DeviceTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/DeviceTest.cs index ab4b89de2c9..18f022cbc05 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/DeviceTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/DeviceTest.cs @@ -75,7 +75,7 @@ public void DeviceSetup () } catch (Exception ex) { Console.Error.WriteLine ("Failed to determine whether there is Android target emulator or not: " + ex); } - SetAdbLogcatBufferSize (64); + SetAdbLogcatBufferSize (128); CreateGuestUser (GuestUserName); } } @@ -396,13 +396,13 @@ protected static string GetAttachedDeviceSerial () return serial.Trim (); } - protected static string [] GetOverrideDirectoryPaths (string packageName) + protected static string [] GetOverrideDirectoryPaths (string packageName, string abi) { return new string [] { - $"/data/data/{packageName}/files/.__override__", - $"/storage/emulated/0/Android/data/{packageName}/files/.__override__", - $"/mnt/shell/emulated/0/Android/data/{packageName}/files/.__override__", - $"/storage/sdcard/Android/data/{packageName}/files/.__override__", + $"/data/data/{packageName}/files/.__override__/{abi}", + $"/storage/emulated/0/Android/data/{packageName}/files/.__override__/{abi}", + $"/mnt/shell/emulated/0/Android/data/{packageName}/files/.__override__/{abi}", + $"/storage/sdcard/Android/data/{packageName}/files/.__override__/{abi}", }; } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj index 3411f1480e5..749318c47ab 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj @@ -32,7 +32,7 @@ - + ..\Expected\GenerateDesignerFileExpected.cs PreserveNewest diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs index c583e2b6192..8407a3af198 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs @@ -10,6 +10,8 @@ using System.Xml.XPath; using System.Xml.Linq; +using Xamarin.Android.Tasks; + namespace Xamarin.ProjectTools { public class Builder : IDisposable @@ -20,6 +22,8 @@ public class Builder : IDisposable string root; string buildLogFullPath; + IEnumerable? lastBuildOutput; + public bool IsUnix { get; set; } /// /// This passes /p:BuildingInsideVisualStudio=True, command-line to MSBuild @@ -32,10 +36,17 @@ public class Builder : IDisposable public LoggerVerbosity Verbosity { get; set; } = LoggerVerbosity.Diagnostic; public IEnumerable LastBuildOutput { get { + if (lastBuildOutput != null) { + return lastBuildOutput; + } + if (!string.IsNullOrEmpty (buildLogFullPath) && File.Exists (buildLogFullPath)) { - return File.ReadLines (buildLogFullPath, Encoding.UTF8); + lastBuildOutput = File.ReadLines (buildLogFullPath, Encoding.UTF8); + } else { + lastBuildOutput = Enumerable.Empty (); } - return Enumerable.Empty (); + + return lastBuildOutput; } } public TimeSpan LastBuildTime { get; protected set; } @@ -120,6 +131,39 @@ public void GetTargetFrameworkVersionRange (out string firstApiLevel, out string allFrameworkVersions = allTFVs.ToArray (); } + public HashSet GetBuildRuntimeIdentifiers () + { + var ret = new HashSet (StringComparer.OrdinalIgnoreCase); + foreach (string l in LastBuildOutput) { + string line = l.Trim (); + if (line.Length == 0 || line[0] != 'R') { + continue; + } + + // Here's hoping MSBuild doesn't change the property reporting format + if (!line.StartsWith ("RuntimeIdentifiers =", StringComparison.Ordinal)) { + continue; + } + + foreach (string r in line.Split ('=')[1].Split (';')) { + ret.Add (r.Trim ()); + } + break; + } + + return ret; + } + + public HashSet GetBuildAbis () + { + var ret = new HashSet (StringComparer.OrdinalIgnoreCase); + foreach (string rid in GetBuildRuntimeIdentifiers ()) { + ret.Add (MonoAndroidHelper.RidToAbi (rid)); + } + + return ret; + } + static string GetApiInfoElementValue (string androidApiInfo, string elementPath) { if (!File.Exists (androidApiInfo)) @@ -159,6 +203,7 @@ protected virtual void Dispose (bool disposing) protected bool BuildInternal (string projectOrSolution, string target, string [] parameters = null, Dictionary environmentVariables = null, bool restore = true, string binlogName = "msbuild") { + lastBuildOutput = null; // make sure we don't return the previous build's cached output buildLogFullPath = (!string.IsNullOrEmpty (BuildLogFile)) ? Path.GetFullPath (Path.Combine (XABuildPaths.TestOutputDirectory, Path.GetDirectoryName (projectOrSolution), BuildLogFile)) : null; @@ -372,4 +417,3 @@ string QuoteFileName (string fileName) } } - diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc index 7bd5cb51f7c..5886262970d 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc @@ -4,50 +4,50 @@ "AndroidManifest.xml": { "Size": 3036 }, - "assemblies/_Microsoft.Android.Resource.Designer.dll": { - "Size": 1028 + "classes.dex": { + "Size": 377684 }, - "assemblies/Java.Interop.dll": { - "Size": 61587 + "lib/arm64-v8a/lib__Microsoft.Android.Resource.Designer.dll.so": { + "Size": 1027 }, - "assemblies/Mono.Android.dll": { - "Size": 90798 + "lib/arm64-v8a/lib_Java.Interop.dll.so": { + "Size": 63889 }, - "assemblies/Mono.Android.Runtime.dll": { - "Size": 5225 + "lib/arm64-v8a/lib_Mono.Android.dll.so": { + "Size": 90449 }, - "assemblies/rc.bin": { - "Size": 1512 + "lib/arm64-v8a/lib_Mono.Android.Runtime.dll.so": { + "Size": 5145 }, - "assemblies/System.Console.dll": { - "Size": 6546 + "lib/arm64-v8a/lib_System.Console.dll.so": { + "Size": 6541 }, - "assemblies/System.Linq.dll": { - "Size": 8553 + "lib/arm64-v8a/lib_System.Linq.dll.so": { + "Size": 8647 }, - "assemblies/System.Private.CoreLib.dll": { - "Size": 553721 + "lib/arm64-v8a/lib_System.Private.CoreLib.dll.so": { + "Size": 553289 }, - "assemblies/System.Runtime.dll": { - "Size": 2550 + "lib/arm64-v8a/lib_System.Runtime.dll.so": { + "Size": 2545 }, - "assemblies/System.Runtime.InteropServices.dll": { - "Size": 4026 + "lib/arm64-v8a/lib_System.Runtime.InteropServices.dll.so": { + "Size": 4022 }, - "assemblies/UnnamedProject.dll": { - "Size": 2936 + "lib/arm64-v8a/lib_UnnamedProject.dll.so": { + "Size": 2932 }, - "classes.dex": { - "Size": 377856 + "lib/arm64-v8a/libarc.bin.so": { + "Size": 1512 }, "lib/arm64-v8a/libmono-component-marshal-ilgen.so": { "Size": 87080 }, "lib/arm64-v8a/libmonodroid.so": { - "Size": 339864 + "Size": 355872 }, "lib/arm64-v8a/libmonosgen-2.0.so": { - "Size": 3186504 + "Size": 3185656 }, "lib/arm64-v8a/libSystem.IO.Compression.Native.so": { "Size": 723560 @@ -59,16 +59,16 @@ "Size": 155560 }, "lib/arm64-v8a/libxamarin-app.so": { - "Size": 11680 + "Size": 12336 }, "META-INF/BNDLTOOL.RSA": { - "Size": 1223 + "Size": 1221 }, "META-INF/BNDLTOOL.SF": { - "Size": 3037 + "Size": 3147 }, "META-INF/MANIFEST.MF": { - "Size": 2910 + "Size": 3020 }, "res/drawable-hdpi-v4/icon.png": { "Size": 2178 @@ -95,5 +95,5 @@ "Size": 1904 } }, - "PackageSize": 2783562 + "PackageSize": 2668984 } \ No newline at end of file diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc index 58f281d0e0c..015199489b9 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc @@ -4,242 +4,242 @@ "AndroidManifest.xml": { "Size": 6652 }, - "assemblies/_Microsoft.Android.Resource.Designer.dll": { + "classes.dex": { + "Size": 9418292 + }, + "classes2.dex": { + "Size": 150904 + }, + "kotlin/annotation/annotation.kotlin_builtins": { + "Size": 928 + }, + "kotlin/collections/collections.kotlin_builtins": { + "Size": 3685 + }, + "kotlin/coroutines/coroutines.kotlin_builtins": { + "Size": 200 + }, + "kotlin/internal/internal.kotlin_builtins": { + "Size": 646 + }, + "kotlin/kotlin.kotlin_builtins": { + "Size": 18640 + }, + "kotlin/ranges/ranges.kotlin_builtins": { + "Size": 3399 + }, + "kotlin/reflect/reflect.kotlin_builtins": { + "Size": 2396 + }, + "lib/arm64-v8a/lib__Microsoft.Android.Resource.Designer.dll.so": { "Size": 2279 }, - "assemblies/FormsViewGroup.dll": { + "lib/arm64-v8a/lib_FormsViewGroup.dll.so": { "Size": 8090 }, - "assemblies/Java.Interop.dll": { - "Size": 72297 - }, - "assemblies/Mono.Android.dll": { - "Size": 456687 + "lib/arm64-v8a/lib_Java.Interop.dll.so": { + "Size": 72360 }, - "assemblies/Mono.Android.Runtime.dll": { - "Size": 5143 + "lib/arm64-v8a/lib_Mono.Android.dll.so": { + "Size": 456663 }, - "assemblies/mscorlib.dll": { - "Size": 3999 + "lib/arm64-v8a/lib_Mono.Android.Runtime.dll.so": { + "Size": 5145 }, - "assemblies/netstandard.dll": { - "Size": 5632 + "lib/arm64-v8a/lib_mscorlib.dll.so": { + "Size": 3992 }, - "assemblies/rc.bin": { - "Size": 1512 + "lib/arm64-v8a/lib_netstandard.dll.so": { + "Size": 5625 }, - "assemblies/System.Collections.Concurrent.dll": { - "Size": 11523 + "lib/arm64-v8a/lib_System.Collections.Concurrent.dll.so": { + "Size": 11520 }, - "assemblies/System.Collections.dll": { - "Size": 15415 + "lib/arm64-v8a/lib_System.Collections.dll.so": { + "Size": 15411 }, - "assemblies/System.Collections.NonGeneric.dll": { - "Size": 7445 + "lib/arm64-v8a/lib_System.Collections.NonGeneric.dll.so": { + "Size": 7442 }, - "assemblies/System.ComponentModel.dll": { - "Size": 1941 + "lib/arm64-v8a/lib_System.ComponentModel.dll.so": { + "Size": 1935 }, - "assemblies/System.ComponentModel.Primitives.dll": { - "Size": 2555 + "lib/arm64-v8a/lib_System.ComponentModel.Primitives.dll.so": { + "Size": 2549 }, - "assemblies/System.ComponentModel.TypeConverter.dll": { - "Size": 6089 + "lib/arm64-v8a/lib_System.ComponentModel.TypeConverter.dll.so": { + "Size": 6084 }, - "assemblies/System.Console.dll": { - "Size": 6582 + "lib/arm64-v8a/lib_System.Console.dll.so": { + "Size": 6576 }, - "assemblies/System.Core.dll": { - "Size": 1976 + "lib/arm64-v8a/lib_System.Core.dll.so": { + "Size": 1969 }, - "assemblies/System.Diagnostics.DiagnosticSource.dll": { - "Size": 9064 + "lib/arm64-v8a/lib_System.Diagnostics.DiagnosticSource.dll.so": { + "Size": 9060 }, - "assemblies/System.Diagnostics.TraceSource.dll": { - "Size": 6547 + "lib/arm64-v8a/lib_System.Diagnostics.TraceSource.dll.so": { + "Size": 6543 }, - "assemblies/System.dll": { - "Size": 2333 + "lib/arm64-v8a/lib_System.dll.so": { + "Size": 2326 }, - "assemblies/System.Drawing.dll": { - "Size": 1937 + "lib/arm64-v8a/lib_System.Drawing.dll.so": { + "Size": 1932 }, - "assemblies/System.Drawing.Primitives.dll": { - "Size": 11966 + "lib/arm64-v8a/lib_System.Drawing.Primitives.dll.so": { + "Size": 11965 }, - "assemblies/System.IO.Compression.Brotli.dll": { - "Size": 11192 + "lib/arm64-v8a/lib_System.IO.Compression.Brotli.dll.so": { + "Size": 11187 }, - "assemblies/System.IO.Compression.dll": { - "Size": 15868 + "lib/arm64-v8a/lib_System.IO.Compression.dll.so": { + "Size": 15864 }, - "assemblies/System.IO.IsolatedStorage.dll": { - "Size": 9899 + "lib/arm64-v8a/lib_System.IO.IsolatedStorage.dll.so": { + "Size": 9895 }, - "assemblies/System.Linq.dll": { - "Size": 20517 + "lib/arm64-v8a/lib_System.Linq.dll.so": { + "Size": 20514 }, - "assemblies/System.Linq.Expressions.dll": { - "Size": 164631 + "lib/arm64-v8a/lib_System.Linq.Expressions.dll.so": { + "Size": 164629 }, - "assemblies/System.Net.Http.dll": { - "Size": 67564 + "lib/arm64-v8a/lib_System.Net.Http.dll.so": { + "Size": 67561 }, - "assemblies/System.Net.Primitives.dll": { - "Size": 22363 + "lib/arm64-v8a/lib_System.Net.Primitives.dll.so": { + "Size": 22354 }, - "assemblies/System.Net.Requests.dll": { - "Size": 3594 + "lib/arm64-v8a/lib_System.Net.Requests.dll.so": { + "Size": 3588 }, - "assemblies/System.ObjectModel.dll": { - "Size": 8572 + "lib/arm64-v8a/lib_System.ObjectModel.dll.so": { + "Size": 8565 }, - "assemblies/System.Private.CoreLib.dll": { - "Size": 869622 + "lib/arm64-v8a/lib_System.Private.CoreLib.dll.so": { + "Size": 870644 }, - "assemblies/System.Private.DataContractSerialization.dll": { - "Size": 193441 + "lib/arm64-v8a/lib_System.Private.DataContractSerialization.dll.so": { + "Size": 193443 }, - "assemblies/System.Private.Uri.dll": { - "Size": 42907 + "lib/arm64-v8a/lib_System.Private.Uri.dll.so": { + "Size": 42904 }, - "assemblies/System.Private.Xml.dll": { - "Size": 216025 + "lib/arm64-v8a/lib_System.Private.Xml.dll.so": { + "Size": 216020 }, - "assemblies/System.Private.Xml.Linq.dll": { - "Size": 16627 + "lib/arm64-v8a/lib_System.Private.Xml.Linq.dll.so": { + "Size": 16622 }, - "assemblies/System.Runtime.dll": { - "Size": 2709 + "lib/arm64-v8a/lib_System.Runtime.dll.so": { + "Size": 2703 }, - "assemblies/System.Runtime.InteropServices.dll": { + "lib/arm64-v8a/lib_System.Runtime.InteropServices.dll.so": { "Size": 4022 }, - "assemblies/System.Runtime.Serialization.dll": { - "Size": 1865 + "lib/arm64-v8a/lib_System.Runtime.Serialization.dll.so": { + "Size": 1859 }, - "assemblies/System.Runtime.Serialization.Formatters.dll": { - "Size": 2485 + "lib/arm64-v8a/lib_System.Runtime.Serialization.Formatters.dll.so": { + "Size": 2479 }, - "assemblies/System.Runtime.Serialization.Primitives.dll": { - "Size": 3757 + "lib/arm64-v8a/lib_System.Runtime.Serialization.Primitives.dll.so": { + "Size": 3751 }, - "assemblies/System.Security.Cryptography.dll": { - "Size": 8102 + "lib/arm64-v8a/lib_System.Security.Cryptography.dll.so": { + "Size": 8099 }, - "assemblies/System.Text.RegularExpressions.dll": { - "Size": 159848 + "lib/arm64-v8a/lib_System.Text.RegularExpressions.dll.so": { + "Size": 161328 }, - "assemblies/System.Xml.dll": { - "Size": 1760 + "lib/arm64-v8a/lib_System.Xml.dll.so": { + "Size": 1754 }, - "assemblies/System.Xml.Linq.dll": { - "Size": 1775 + "lib/arm64-v8a/lib_System.Xml.Linq.dll.so": { + "Size": 1768 }, - "assemblies/UnnamedProject.dll": { + "lib/arm64-v8a/lib_UnnamedProject.dll.so": { "Size": 5007 }, - "assemblies/Xamarin.AndroidX.Activity.dll": { + "lib/arm64-v8a/lib_Xamarin.AndroidX.Activity.dll.so": { "Size": 16116 }, - "assemblies/Xamarin.AndroidX.AppCompat.AppCompatResources.dll": { + "lib/arm64-v8a/lib_Xamarin.AndroidX.AppCompat.AppCompatResources.dll.so": { "Size": 6216 }, - "assemblies/Xamarin.AndroidX.AppCompat.dll": { + "lib/arm64-v8a/lib_Xamarin.AndroidX.AppCompat.dll.so": { "Size": 138025 }, - "assemblies/Xamarin.AndroidX.CardView.dll": { + "lib/arm64-v8a/lib_Xamarin.AndroidX.CardView.dll.so": { "Size": 6959 }, - "assemblies/Xamarin.AndroidX.CoordinatorLayout.dll": { + "lib/arm64-v8a/lib_Xamarin.AndroidX.CoordinatorLayout.dll.so": { "Size": 17921 }, - "assemblies/Xamarin.AndroidX.Core.dll": { + "lib/arm64-v8a/lib_Xamarin.AndroidX.Core.dll.so": { "Size": 126882 }, - "assemblies/Xamarin.AndroidX.CursorAdapter.dll": { + "lib/arm64-v8a/lib_Xamarin.AndroidX.CursorAdapter.dll.so": { "Size": 8978 }, - "assemblies/Xamarin.AndroidX.DrawerLayout.dll": { + "lib/arm64-v8a/lib_Xamarin.AndroidX.DrawerLayout.dll.so": { "Size": 15286 }, - "assemblies/Xamarin.AndroidX.Fragment.dll": { + "lib/arm64-v8a/lib_Xamarin.AndroidX.Fragment.dll.so": { "Size": 51498 }, - "assemblies/Xamarin.AndroidX.Legacy.Support.Core.UI.dll": { + "lib/arm64-v8a/lib_Xamarin.AndroidX.Legacy.Support.Core.UI.dll.so": { "Size": 6233 }, - "assemblies/Xamarin.AndroidX.Lifecycle.Common.dll": { + "lib/arm64-v8a/lib_Xamarin.AndroidX.Lifecycle.Common.dll.so": { "Size": 6890 }, - "assemblies/Xamarin.AndroidX.Lifecycle.LiveData.Core.dll": { + "lib/arm64-v8a/lib_Xamarin.AndroidX.Lifecycle.LiveData.Core.dll.so": { "Size": 6733 }, - "assemblies/Xamarin.AndroidX.Lifecycle.ViewModel.dll": { + "lib/arm64-v8a/lib_Xamarin.AndroidX.Lifecycle.ViewModel.dll.so": { "Size": 7002 }, - "assemblies/Xamarin.AndroidX.Loader.dll": { + "lib/arm64-v8a/lib_Xamarin.AndroidX.Loader.dll.so": { "Size": 13063 }, - "assemblies/Xamarin.AndroidX.RecyclerView.dll": { + "lib/arm64-v8a/lib_Xamarin.AndroidX.RecyclerView.dll.so": { "Size": 93516 }, - "assemblies/Xamarin.AndroidX.SavedState.dll": { + "lib/arm64-v8a/lib_Xamarin.AndroidX.SavedState.dll.so": { "Size": 5107 }, - "assemblies/Xamarin.AndroidX.SwipeRefreshLayout.dll": { + "lib/arm64-v8a/lib_Xamarin.AndroidX.SwipeRefreshLayout.dll.so": { "Size": 13946 }, - "assemblies/Xamarin.AndroidX.ViewPager.dll": { + "lib/arm64-v8a/lib_Xamarin.AndroidX.ViewPager.dll.so": { "Size": 19014 }, - "assemblies/Xamarin.Forms.Core.dll": { + "lib/arm64-v8a/lib_Xamarin.Forms.Core.dll.so": { "Size": 563905 }, - "assemblies/Xamarin.Forms.Platform.Android.dll": { + "lib/arm64-v8a/lib_Xamarin.Forms.Platform.Android.dll.so": { "Size": 373374 }, - "assemblies/Xamarin.Forms.Platform.dll": { + "lib/arm64-v8a/lib_Xamarin.Forms.Platform.dll.so": { "Size": 18753 }, - "assemblies/Xamarin.Forms.Xaml.dll": { + "lib/arm64-v8a/lib_Xamarin.Forms.Xaml.dll.so": { "Size": 63542 }, - "assemblies/Xamarin.Google.Android.Material.dll": { + "lib/arm64-v8a/lib_Xamarin.Google.Android.Material.dll.so": { "Size": 66169 }, - "classes.dex": { - "Size": 9418292 - }, - "classes2.dex": { - "Size": 150904 - }, - "kotlin/annotation/annotation.kotlin_builtins": { - "Size": 928 - }, - "kotlin/collections/collections.kotlin_builtins": { - "Size": 3685 - }, - "kotlin/coroutines/coroutines.kotlin_builtins": { - "Size": 200 - }, - "kotlin/internal/internal.kotlin_builtins": { - "Size": 646 - }, - "kotlin/kotlin.kotlin_builtins": { - "Size": 18640 - }, - "kotlin/ranges/ranges.kotlin_builtins": { - "Size": 3399 - }, - "kotlin/reflect/reflect.kotlin_builtins": { - "Size": 2396 + "lib/arm64-v8a/libarc.bin.so": { + "Size": 1512 }, "lib/arm64-v8a/libmono-component-marshal-ilgen.so": { "Size": 87352 }, "lib/arm64-v8a/libmonodroid.so": { - "Size": 343896 + "Size": 354064 }, "lib/arm64-v8a/libmonosgen-2.0.so": { "Size": 3210968 @@ -254,7 +254,7 @@ "Size": 155568 }, "lib/arm64-v8a/libxamarin-app.so": { - "Size": 115280 + "Size": 118456 }, "META-INF/androidx.activity_activity.version": { "Size": 6 @@ -410,7 +410,7 @@ "Size": 1221 }, "META-INF/BNDLTOOL.SF": { - "Size": 97490 + "Size": 98179 }, "META-INF/com.android.tools/proguard/coroutines.pro": { "Size": 1345 @@ -437,7 +437,7 @@ "Size": 5 }, "META-INF/MANIFEST.MF": { - "Size": 97363 + "Size": 98052 }, "META-INF/maven/com.google.guava/listenablefuture/pom.properties": { "Size": 96 @@ -2477,5 +2477,5 @@ "Size": 812848 } }, - "PackageSize": 10897699 + "PackageSize": 10193867 } \ No newline at end of file diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Utilities/ProjectExtensions.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Utilities/ProjectExtensions.cs index 82891c8360c..3e737019d26 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Utilities/ProjectExtensions.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Utilities/ProjectExtensions.cs @@ -1,5 +1,9 @@ using System; using System.Collections.Generic; +using System.Linq; + +using Xamarin.Android.Tasks; +using Xamarin.Android.Tools; namespace Xamarin.ProjectTools { @@ -17,7 +21,7 @@ public static void SetAndroidSupportedAbis (this IShortFormProject project, para /// /// Sets $(AndroidSupportedAbis) or $(RuntimeIdentifiers) depending if running under dotnet /// - /// A semi-colon-delimited list of ABIs + /// A semi-colon-delimited list of ABIs [Obsolete ("SetAndroidSupportedAbis is deprecated, please use SetRuntimeIdentifiers instead.")] public static void SetAndroidSupportedAbis (this IShortFormProject project, string abis) { @@ -29,7 +33,7 @@ public static void SetRuntimeIdentifier (this IShortFormProject project, string project.SetProperty (KnownProperties.RuntimeIdentifier, AbiUtils.AbiToRuntimeIdentifier (androidAbi)); } - public static void SetRuntimeIdentifiers (this IShortFormProject project, string [] androidAbis) + public static void SetRuntimeIdentifiers (this IShortFormProject project, IEnumerable androidAbis) { var abis = new List (); foreach (var androidAbi in androidAbis) { @@ -37,5 +41,14 @@ public static void SetRuntimeIdentifiers (this IShortFormProject project, string } project.SetProperty (KnownProperties.RuntimeIdentifiers, string.Join (";", abis)); } + + public static void SetRuntimeIdentifiers (this IShortFormProject project, params AndroidTargetArch[] targetArches) + { + if (targetArches == null || targetArches.Length == 0) { + throw new ArgumentException ("must not be null or empty", nameof (targetArches)); + } + + project.SetProperty (KnownProperties.RuntimeIdentifiers, String.Join (";", targetArches.Select (arch => MonoAndroidHelper.ArchToRid (arch)))); + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Xamarin.ProjectTools.csproj b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Xamarin.ProjectTools.csproj index 0d6c4810e2d..2482944cc97 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Xamarin.ProjectTools.csproj +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Xamarin.ProjectTools.csproj @@ -15,6 +15,7 @@ + %(RecursiveDir)appicon.png @@ -27,6 +28,7 @@ + {E34BCFA0-CAA4-412C-AA1C-75DB8D67D157} Xamarin.Android.Tools.AndroidSdk diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ACWMapGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ACWMapGenerator.cs new file mode 100644 index 00000000000..e624c826b31 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ACWMapGenerator.cs @@ -0,0 +1,102 @@ +using System; +using System.Collections.Generic; + +using Java.Interop.Tools.Cecil; +using Java.Interop.Tools.TypeNameMappings; +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Utilities; +using Mono.Cecil; + +namespace Xamarin.Android.Tasks; + +class ACWMapGenerator +{ + readonly TaskLoggingHelper log; + + public ACWMapGenerator (TaskLoggingHelper log) + { + this.log = log; + } + + public bool Generate (NativeCodeGenState codeGenState, string acwMapFile) + { + List javaTypes = codeGenState.JavaTypesForJCW; + TypeDefinitionCache cache = codeGenState.TypeCache; + + // We need to save a map of .NET type -> ACW type for resource file fixups + var managed = new Dictionary (javaTypes.Count, StringComparer.Ordinal); + var java = new Dictionary (javaTypes.Count, StringComparer.Ordinal); + + var managedConflicts = new Dictionary> (0, StringComparer.Ordinal); + var javaConflicts = new Dictionary> (0, StringComparer.Ordinal); + + bool success = true; + + using var acw_map = MemoryStreamPool.Shared.CreateStreamWriter (); + foreach (TypeDefinition type in javaTypes) { + string managedKey = type.FullName.Replace ('/', '.'); + string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'); + + acw_map.Write (type.GetPartialAssemblyQualifiedName (cache)); + acw_map.Write (';'); + acw_map.Write (javaKey); + acw_map.WriteLine (); + + TypeDefinition conflict; + bool hasConflict = false; + if (managed.TryGetValue (managedKey, out conflict)) { + if (!conflict.Module.Name.Equals (type.Module.Name)) { + if (!managedConflicts.TryGetValue (managedKey, out var list)) + managedConflicts.Add (managedKey, list = new List { conflict.GetPartialAssemblyName (cache) }); + list.Add (type.GetPartialAssemblyName (cache)); + success = false; + } + hasConflict = true; + } + if (java.TryGetValue (javaKey, out conflict)) { + if (!conflict.Module.Name.Equals (type.Module.Name)) { + if (!javaConflicts.TryGetValue (javaKey, out var list)) + javaConflicts.Add (javaKey, list = new List { conflict.GetAssemblyQualifiedName (cache) }); + list.Add (type.GetAssemblyQualifiedName (cache)); + success = false; + } + hasConflict = true; + } + if (!hasConflict) { + managed.Add (managedKey, type); + java.Add (javaKey, type); + + acw_map.Write (managedKey); + acw_map.Write (';'); + acw_map.Write (javaKey); + acw_map.WriteLine (); + + acw_map.Write (JavaNativeTypeManager.ToCompatJniName (type, cache).Replace ('/', '.')); + acw_map.Write (';'); + acw_map.Write (javaKey); + acw_map.WriteLine (); + } + } + + acw_map.Flush (); + Files.CopyIfStreamChanged (acw_map.BaseStream, acwMapFile); + + foreach (var kvp in managedConflicts) { + log.LogCodedWarning ("XA4214", Properties.Resources.XA4214, kvp.Key, string.Join (", ", kvp.Value)); + log.LogCodedWarning ("XA4214", Properties.Resources.XA4214_Result, kvp.Key, kvp.Value [0]); + } + + foreach (var kvp in javaConflicts) { + log.LogCodedError ("XA4215", Properties.Resources.XA4215, kvp.Key); + foreach (var typeName in kvp.Value) { + log.LogCodedError ("XA4215", Properties.Resources.XA4215_Details, kvp.Key, typeName); + } + } + + if (javaConflicts.Count > 0) { + return false; + } + + return success; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs index 02b6c6ba776..703209e20a1 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs @@ -40,8 +40,8 @@ sealed class ApplicationConfig public uint system_property_count; public uint number_of_assemblies_in_apk; public uint bundled_assembly_name_width; - public uint number_of_assembly_store_files; public uint number_of_dso_cache_entries; + public uint number_of_shared_libraries; [NativeAssembler (NumberFormat = LLVMIR.LlvmIrVariableNumberFormat.Hexadecimal)] public uint android_runtime_jnienv_class_token; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs index cee11386b4a..8f7e569dfa4 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs @@ -6,7 +6,6 @@ using Java.Interop.Tools.TypeNameMappings; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; -using Microsoft.Android.Build.Tasks; using Xamarin.Android.Tasks.LLVMIR; namespace Xamarin.Android.Tasks @@ -51,6 +50,9 @@ sealed class DSOCacheEntry [NativeAssembler (UsesDataProvider = true, NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal)] public ulong hash; + + [NativeAssembler (NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal)] + public ulong real_name_hash; public bool ignore; [NativeAssembler (UsesDataProvider = true)] @@ -58,6 +60,13 @@ sealed class DSOCacheEntry public IntPtr handle = IntPtr.Zero; } + sealed class DSOApkEntry + { + public ulong name_hash; + public uint offset; // offset into the APK + public int fd; // apk file descriptor + }; + // Order of fields and their type must correspond *exactly* to that in // src/monodroid/jni/xamarin-app.hh AssemblyStoreAssemblyDescriptor structure sealed class AssemblyStoreAssemblyDescriptor @@ -96,6 +105,7 @@ sealed class AssemblyStoreRuntimeData [NativePointer (IsNull = true)] public byte data_start; public uint assembly_count; + public uint index_entry_count; [NativePointer (IsNull = true)] public AssemblyStoreAssemblyDescriptor assemblies; @@ -105,12 +115,16 @@ sealed class XamarinAndroidBundledAssemblyContextDataProvider : NativeAssemblerS { public override ulong GetBufferSize (object data, string fieldName) { - if (String.Compare ("name", fieldName, StringComparison.Ordinal) != 0) { - return 0; + var xaba = EnsureType (data); + if (String.Compare ("name", fieldName, StringComparison.Ordinal) == 0) { + return xaba.name_length; } - var xaba = EnsureType (data); - return xaba.name_length; + if (String.Compare ("file_name", fieldName, StringComparison.Ordinal) == 0) { + return xaba.name_length + MonoAndroidHelper.GetMangledAssemblyNameSizeOverhead (); + } + + return 0; } } @@ -119,7 +133,10 @@ public override ulong GetBufferSize (object data, string fieldName) [NativeAssemblerStructContextDataProvider (typeof (XamarinAndroidBundledAssemblyContextDataProvider))] sealed class XamarinAndroidBundledAssembly { - public int apk_fd; + public int file_fd; + + [NativeAssembler (UsesDataProvider = true), NativePointer (PointsToPreAllocatedBuffer = true)] + public string file_name; public uint data_offset; public uint data_size; @@ -142,6 +159,7 @@ sealed class XamarinAndroidBundledAssembly StructureInfo? applicationConfigStructureInfo; StructureInfo? dsoCacheEntryStructureInfo; + StructureInfo? dsoApkEntryStructureInfo; StructureInfo? xamarinAndroidBundledAssemblyStructureInfo; StructureInfo? assemblyStoreSingleAssemblyRuntimeDataStructureinfo; StructureInfo? assemblyStoreRuntimeDataStructureInfo; @@ -159,7 +177,6 @@ sealed class XamarinAndroidBundledAssembly public bool HaveRuntimeConfigBlob { get; set; } public bool HaveAssemblyStore { get; set; } public int NumberOfAssembliesInApk { get; set; } - public int NumberOfAssemblyStoresInApks { get; set; } public int BundledAssemblyNameWidth { get; set; } // including the trailing NUL public int AndroidRuntimeJNIEnvToken { get; set; } public int JNIEnvInitializeToken { get; set; } @@ -217,8 +234,8 @@ protected override void Construct (LlvmIrModule module) environment_variable_count = (uint)(environmentVariables == null ? 0 : environmentVariables.Count * 2), system_property_count = (uint)(systemProperties == null ? 0 : systemProperties.Count * 2), number_of_assemblies_in_apk = (uint)NumberOfAssembliesInApk, + number_of_shared_libraries = (uint)NativeLibraries.Count, bundled_assembly_name_width = (uint)BundledAssemblyNameWidth, - number_of_assembly_store_files = (uint)NumberOfAssemblyStoresInApks, number_of_dso_cache_entries = (uint)dsoCache.Count, android_runtime_jnienv_class_token = (uint)AndroidRuntimeJNIEnvToken, jnienv_initialize_method_token = (uint)JNIEnvInitializeToken, @@ -237,11 +254,18 @@ protected override void Construct (LlvmIrModule module) }; module.Add (dso_cache); + var dso_apk_entries = new LlvmIrGlobalVariable (typeof(List>), "dso_apk_entries") { + ArrayItemCount = (ulong)NativeLibraries.Count, + Options = LlvmIrVariableOptions.GlobalWritable, + ZeroInitializeArray = true, + }; + module.Add (dso_apk_entries); + if (!HaveAssemblyStore) { xamarinAndroidBundledAssemblies = new List> (NumberOfAssembliesInApk); var emptyBundledAssemblyData = new XamarinAndroidBundledAssembly { - apk_fd = -1, + file_fd = -1, data_offset = 0, data_size = 0, data = 0, @@ -273,19 +297,24 @@ void AddAssemblyStores (LlvmIrModule module) }; module.Add (assembly_store_bundled_assemblies); - itemCount = (ulong)(HaveAssemblyStore ? NumberOfAssemblyStoresInApks : 0); - var assembly_stores = new LlvmIrGlobalVariable (typeof(List>), "assembly_stores", LlvmIrVariableOptions.GlobalWritable) { - ZeroInitializeArray = true, - ArrayItemCount = itemCount, + var storeRuntimeData = new AssemblyStoreRuntimeData { + data_start = 0, + assembly_count = 0, }; - module.Add (assembly_stores); + + var assembly_store = new LlvmIrGlobalVariable ( + new StructureInstance(assemblyStoreRuntimeDataStructureInfo, storeRuntimeData), + "assembly_store", + LlvmIrVariableOptions.GlobalWritable + ); + module.Add (assembly_store); } void HashAndSortDSOCache (LlvmIrVariable variable, LlvmIrModuleTarget target, object? state) { var cache = variable.Value as List>; if (cache == null) { - throw new InvalidOperationException ($"Internal error: DSO cache must no be empty"); + throw new InvalidOperationException ($"Internal error: DSO cache must not be empty"); } bool is64Bit = target.Is64Bit; @@ -300,6 +329,7 @@ void HashAndSortDSOCache (LlvmIrVariable variable, LlvmIrModuleTarget target, ob } entry.hash = MonoAndroidHelper.GetXxHash (entry.HashedName, is64Bit); + entry.real_name_hash = MonoAndroidHelper.GetXxHash (entry.name, is64Bit); } cache.Sort ((StructureInstance a, StructureInstance b) => a.Instance.hash.CompareTo (b.Instance.hash)); @@ -350,7 +380,14 @@ void AddNameMutations (string name) { nameMutations.Add (name); if (name.EndsWith (".dll.so", StringComparison.OrdinalIgnoreCase)) { - nameMutations.Add (Path.GetFileNameWithoutExtension (Path.GetFileNameWithoutExtension (name))!); + string nameNoExt = Path.GetFileNameWithoutExtension (Path.GetFileNameWithoutExtension (name))!; + nameMutations.Add (nameNoExt); + + // This helps us at runtime, because sometimes MonoVM will ask for "AssemblyName" and sometimes for "AssemblyName.dll". + // In the former case, the runtime would ask for the "libaot-AssemblyName.so" image, which doesn't exist - we have + // "libaot-AssemblyName.dll.so" instead and, thus, we are forced to check for and append the missing ".dll" extension when + // loading the assembly, unnecessarily wasting time. + nameMutations.Add ($"{nameNoExt}.so"); } else { nameMutations.Add (Path.GetFileNameWithoutExtension (name)!); } @@ -375,6 +412,7 @@ void MapStructures (LlvmIrModule module) assemblyStoreRuntimeDataStructureInfo = module.MapStructure (); xamarinAndroidBundledAssemblyStructureInfo = module.MapStructure (); dsoCacheEntryStructureInfo = module.MapStructure (); + dsoApkEntryStructureInfo = module.MapStructure (); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigTaskState.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigTaskState.cs deleted file mode 100644 index 5cdd3a6e6a2..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigTaskState.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Xamarin.Android.Tasks -{ - class ApplicationConfigTaskState - { - public const string RegisterTaskObjectKey = "Xamarin.Android.Tasks.ApplicationConfigTaskState"; - - public bool JniAddNativeMethodRegistrationAttributePresent { get; set; } = false; - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs deleted file mode 100644 index a5b5811b7de..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs +++ /dev/null @@ -1,112 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -namespace Xamarin.Android.Tasks -{ - class ArchAssemblyStore : AssemblyStore - { - readonly Dictionary> assemblies; - HashSet seenArchAssemblyNames; - - public ArchAssemblyStore (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyStoreGlobalIndex globalIndexCounter) - : base (apkName, archiveAssembliesPrefix, log, id, globalIndexCounter) - { - assemblies = new Dictionary> (StringComparer.OrdinalIgnoreCase); - } - - public override string WriteIndex (List globalIndex) - { - throw new InvalidOperationException ("Architecture-specific assembly blob cannot contain global assembly index"); - } - - public override void Add (AssemblyStoreAssemblyInfo blobAssembly) - { - if (String.IsNullOrEmpty (blobAssembly.Abi)) { - throw new InvalidOperationException ($"Architecture-agnostic assembly cannot be added to an architecture-specific blob ({blobAssembly.FilesystemAssemblyPath})"); - } - - if (!assemblies.ContainsKey (blobAssembly.Abi)) { - assemblies.Add (blobAssembly.Abi, new List ()); - } - - List blobAssemblies = assemblies[blobAssembly.Abi]; - blobAssemblies.Add (blobAssembly); - - if (seenArchAssemblyNames == null) { - seenArchAssemblyNames = new HashSet (StringComparer.Ordinal); - } - - string assemblyName = GetAssemblyName (blobAssembly); - if (seenArchAssemblyNames.Contains (assemblyName)) { - return; - } - - seenArchAssemblyNames.Add (assemblyName); - } - - public override void Generate (string outputDirectory, List globalIndex, List blobPaths) - { - if (assemblies.Count == 0) { - return; - } - - var assemblyNames = new Dictionary (); - foreach (var kvp in assemblies) { - string abi = kvp.Key; - List archAssemblies = kvp.Value; - - // All the architecture blobs must have assemblies in exactly the same order - archAssemblies.Sort ((AssemblyStoreAssemblyInfo a, AssemblyStoreAssemblyInfo b) => Path.GetFileName (a.FilesystemAssemblyPath).CompareTo (Path.GetFileName (b.FilesystemAssemblyPath))); - if (assemblyNames.Count == 0) { - for (int i = 0; i < archAssemblies.Count; i++) { - AssemblyStoreAssemblyInfo info = archAssemblies[i]; - assemblyNames.Add (i, Path.GetFileName (info.FilesystemAssemblyPath)); - } - continue; - } - - if (archAssemblies.Count != assemblyNames.Count) { - throw new InvalidOperationException ($"Assembly list for ABI '{abi}' has a different number of assemblies than other ABI lists (expected {assemblyNames.Count}, found {archAssemblies.Count}"); - } - - for (int i = 0; i < archAssemblies.Count; i++) { - AssemblyStoreAssemblyInfo info = archAssemblies[i]; - string fileName = Path.GetFileName (info.FilesystemAssemblyPath); - - if (assemblyNames[i] != fileName) { - throw new InvalidOperationException ($"Assembly list for ABI '{abi}' differs from other lists at index {i}. Expected '{assemblyNames[i]}', found '{fileName}'"); - } - } - } - - bool addToGlobalIndex = true; - foreach (var kvp in assemblies) { - string abi = kvp.Key; - List archAssemblies = kvp.Value; - - if (archAssemblies.Count == 0) { - continue; - } - - // Android uses underscores in place of dashes in ABI names, let's follow the convention - string androidAbi = abi.Replace ('-', '_'); - Generate (Path.Combine (outputDirectory, $"{ApkName}_{BlobPrefix}.{androidAbi}{BlobExtension}"), archAssemblies, globalIndex, blobPaths, addToGlobalIndex); - - // NOTE: not thread safe! The counter must grow monotonically but we also don't want to use different index values for the architecture-specific - // assemblies with the same names, that would only waste space in the generated `libxamarin-app.so`. To use the same index values for the same - // assemblies in different architectures we need to move the counter back here. - GlobalIndexCounter.Subtract ((uint)archAssemblies.Count); - - if (addToGlobalIndex) { - // We want the architecture-specific assemblies to be added to the global index only once - addToGlobalIndex = false; - } - } - - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs deleted file mode 100644 index 378a9ee8c46..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs +++ /dev/null @@ -1,377 +0,0 @@ -using System; -using System.Buffers; -using System.Collections.Generic; -using System.IO; -using System.Text; - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -namespace Xamarin.Android.Tasks -{ - abstract class AssemblyStore - { - // The two constants below must match their counterparts in src/monodroid/jni/xamarin-app.hh - const uint BlobMagic = 0x41424158; // 'XABA', little-endian, must match the BUNDLED_ASSEMBLIES_BLOB_MAGIC native constant - const uint BlobVersion = 1; // Must match the BUNDLED_ASSEMBLIES_BLOB_VERSION native constant - - // MUST be equal to the size of the BlobBundledAssembly struct in src/monodroid/jni/xamarin-app.hh - const uint BlobBundledAssemblyNativeStructSize = 6 * sizeof (uint); - - // MUST be equal to the size of the BlobHashEntry struct in src/monodroid/jni/xamarin-app.hh - const uint BlobHashEntryNativeStructSize = sizeof (ulong) + (3 * sizeof (uint)); - - // MUST be equal to the size of the BundledAssemblyBlobHeader struct in src/monodroid/jni/xamarin-app.hh - const uint BlobHeaderNativeStructSize = sizeof (uint) * 5; - - protected const string BlobPrefix = "assemblies"; - protected const string BlobExtension = ".blob"; - - static readonly ArrayPool bytePool = ArrayPool.Shared; - - string archiveAssembliesPrefix; - string indexBlobPath; - - protected string ApkName { get; } - protected TaskLoggingHelper Log { get; } - protected AssemblyStoreGlobalIndex GlobalIndexCounter { get; } - - public uint ID { get; } - public bool IsIndexStore => ID == 0; - - protected AssemblyStore (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyStoreGlobalIndex globalIndexCounter) - { - if (String.IsNullOrEmpty (archiveAssembliesPrefix)) { - throw new ArgumentException ("must not be null or empty", nameof (archiveAssembliesPrefix)); - } - - if (String.IsNullOrEmpty (apkName)) { - throw new ArgumentException ("must not be null or empty", nameof (apkName)); - } - - GlobalIndexCounter = globalIndexCounter ?? throw new ArgumentNullException (nameof (globalIndexCounter)); - ID = id; - - this.archiveAssembliesPrefix = archiveAssembliesPrefix; - ApkName = apkName; - Log = log; - } - - public abstract void Add (AssemblyStoreAssemblyInfo blobAssembly); - public abstract void Generate (string outputDirectory, List globalIndex, List blobPaths); - - public virtual string WriteIndex (List globalIndex) - { - if (!IsIndexStore) { - throw new InvalidOperationException ("Assembly index may be written only to blob with index 0"); - } - - if (String.IsNullOrEmpty (indexBlobPath)) { - throw new InvalidOperationException ("Index blob path not set, was Generate called properly?"); - } - - if (globalIndex == null) { - throw new ArgumentNullException (nameof (globalIndex)); - } - - string indexBlobHeaderPath = $"{indexBlobPath}.hdr"; - string indexBlobManifestPath = Path.ChangeExtension (indexBlobPath, "manifest"); - - using (var hfs = File.Open (indexBlobHeaderPath, FileMode.Create, FileAccess.Write, FileShare.None)) { - using (var writer = new BinaryWriter (hfs, Encoding.UTF8, leaveOpen: true)) { - WriteIndex (writer, indexBlobManifestPath, globalIndex); - writer.Flush (); - } - - using (var ifs = File.Open (indexBlobPath, FileMode.Open, FileAccess.Read, FileShare.Read)) { - ifs.CopyTo (hfs); - hfs.Flush (); - } - } - - File.Delete (indexBlobPath); - File.Move (indexBlobHeaderPath, indexBlobPath); - - return indexBlobManifestPath; - } - - void WriteIndex (BinaryWriter blobWriter, string manifestPath, List globalIndex) - { - using (var manifest = File.Open (manifestPath, FileMode.Create, FileAccess.Write)) { - using (var manifestWriter = new StreamWriter (manifest, new UTF8Encoding (false))) { - WriteIndex (blobWriter, manifestWriter, globalIndex); - manifestWriter.Flush (); - } - } - } - - void WriteIndex (BinaryWriter blobWriter, StreamWriter manifestWriter, List globalIndex) - { - uint localEntryCount = 0; - var localAssemblies = new List (); - - manifestWriter.WriteLine ("Hash 32 Hash 64 Blob ID Blob idx Name"); - - var seenHashes32 = new HashSet (); - var seenHashes64 = new HashSet (); - bool haveDuplicates = false; - foreach (AssemblyStoreIndexEntry assembly in globalIndex) { - if (assembly.StoreID == ID) { - localEntryCount++; - localAssemblies.Add (assembly); - } - - if (WarnAboutDuplicateHash ("32", assembly.Name, assembly.NameHash32, seenHashes32) || - WarnAboutDuplicateHash ("64", assembly.Name, assembly.NameHash64, seenHashes64)) { - haveDuplicates = true; - } - - manifestWriter.WriteLine ($"0x{assembly.NameHash32:x08} 0x{assembly.NameHash64:x016} {assembly.StoreID:d03} {assembly.LocalBlobIndex:d04} {assembly.Name}"); - } - - if (haveDuplicates) { - throw new InvalidOperationException ("Duplicate assemblies encountered"); - } - - uint globalAssemblyCount = (uint)globalIndex.Count; - - blobWriter.Seek (0, SeekOrigin.Begin); - WriteBlobHeader (blobWriter, localEntryCount, globalAssemblyCount); - - // Header and two tables of the same size, each for 32 and 64-bit hashes - uint offsetFixup = BlobHeaderNativeStructSize + (BlobHashEntryNativeStructSize * globalAssemblyCount * 2); - - WriteAssemblyDescriptors (blobWriter, localAssemblies, CalculateOffsetFixup ((uint)localAssemblies.Count, offsetFixup)); - - var sortedIndex = new List (globalIndex); - sortedIndex.Sort ((AssemblyStoreIndexEntry a, AssemblyStoreIndexEntry b) => a.NameHash32.CompareTo (b.NameHash32)); - foreach (AssemblyStoreIndexEntry entry in sortedIndex) { - WriteHash (entry, entry.NameHash32); - } - - sortedIndex.Sort ((AssemblyStoreIndexEntry a, AssemblyStoreIndexEntry b) => a.NameHash64.CompareTo (b.NameHash64)); - foreach (AssemblyStoreIndexEntry entry in sortedIndex) { - WriteHash (entry, entry.NameHash64); - } - - void WriteHash (AssemblyStoreIndexEntry entry, ulong hash) - { - blobWriter.Write (hash); - blobWriter.Write (entry.MappingIndex); - blobWriter.Write (entry.LocalBlobIndex); - blobWriter.Write (entry.StoreID); - } - - bool WarnAboutDuplicateHash (string bitness, string assemblyName, ulong hash, HashSet seenHashes) - { - if (seenHashes.Contains (hash)) { - Log.LogMessage (MessageImportance.High, $"Duplicate {bitness}-bit hash 0x{hash} encountered for assembly {assemblyName}"); - return true; - } - - seenHashes.Add (hash); - return false; - } - } - - protected string GetAssemblyName (AssemblyStoreAssemblyInfo assembly) - { - string assemblyName = Path.GetFileNameWithoutExtension (assembly.FilesystemAssemblyPath); - if (assemblyName.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) { - assemblyName = Path.GetFileNameWithoutExtension (assemblyName); - } - - return assemblyName; - } - - protected void Generate (string outputFilePath, List assemblies, List globalIndex, List blobPaths, bool addToGlobalIndex = true) - { - if (globalIndex == null) { - throw new ArgumentNullException (nameof (globalIndex)); - } - - if (blobPaths == null) { - throw new ArgumentNullException (nameof (blobPaths)); - } - - if (IsIndexStore) { - indexBlobPath = outputFilePath; - } - - blobPaths.Add (outputFilePath); - Log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: generating blob: {outputFilePath}"); - - using (var fs = File.Open (outputFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { - using (var writer = new BinaryWriter (fs, Encoding.UTF8)) { - Generate (writer, assemblies, globalIndex, addToGlobalIndex); - writer.Flush (); - } - } - } - - void Generate (BinaryWriter writer, List assemblies, List globalIndex, bool addToGlobalIndex) - { - var localAssemblies = new List (); - - if (!IsIndexStore) { - // Index blob's header and data before the assemblies is handled in WriteIndex in a slightly different - // way. - uint nbytes = BlobHeaderNativeStructSize + (BlobBundledAssemblyNativeStructSize * (uint)assemblies.Count); - var zeros = bytePool.Rent ((int)nbytes); - writer.Write (zeros, 0, (int)nbytes); - bytePool.Return (zeros); - } - - foreach (AssemblyStoreAssemblyInfo assembly in assemblies) { - string assemblyName = GetAssemblyName (assembly); - string archivePath = assembly.ArchiveAssemblyPath; - if (archivePath.StartsWith (archiveAssembliesPrefix, StringComparison.OrdinalIgnoreCase)) { - archivePath = archivePath.Substring (archiveAssembliesPrefix.Length); - } - - if (!String.IsNullOrEmpty (assembly.Abi)) { - string abiPath = $"{assembly.Abi}/"; - if (archivePath.StartsWith (abiPath, StringComparison.Ordinal)) { - archivePath = archivePath.Substring (abiPath.Length); - } - } - - if (!String.IsNullOrEmpty (archivePath)) { - if (archivePath.EndsWith ("/", StringComparison.Ordinal)) { - assemblyName = $"{archivePath}{assemblyName}"; - } else { - assemblyName = $"{archivePath}/{assemblyName}"; - } - } - - AssemblyStoreIndexEntry entry = WriteAssembly (writer, assembly, assemblyName, (uint)localAssemblies.Count); - if (addToGlobalIndex) { - globalIndex.Add (entry); - } - localAssemblies.Add (entry); - } - - writer.Flush (); - - if (IsIndexStore) { - return; - } - - writer.Seek (0, SeekOrigin.Begin); - WriteBlobHeader (writer, (uint)localAssemblies.Count); - WriteAssemblyDescriptors (writer, localAssemblies); - } - - uint CalculateOffsetFixup (uint localAssemblyCount, uint extraOffset = 0) - { - return (BlobBundledAssemblyNativeStructSize * (uint)localAssemblyCount) + extraOffset; - } - - void WriteBlobHeader (BinaryWriter writer, uint localEntryCount, uint globalEntryCount = 0) - { - // Header, must be identical to the BundledAssemblyBlobHeader structure in src/monodroid/jni/xamarin-app.hh - writer.Write (BlobMagic); // magic - writer.Write (BlobVersion); // version - writer.Write (localEntryCount); // local_entry_count - writer.Write (globalEntryCount); // global_entry_count - writer.Write ((uint)ID); // blob_id - } - - void WriteAssemblyDescriptors (BinaryWriter writer, List assemblies, uint offsetFixup = 0) - { - // Each assembly must be identical to the BlobBundledAssembly structure in src/monodroid/jni/xamarin-app.hh - - foreach (AssemblyStoreIndexEntry assembly in assemblies) { - AdjustOffsets (assembly, offsetFixup); - - writer.Write (assembly.DataOffset); - writer.Write (assembly.DataSize); - - writer.Write (assembly.DebugDataOffset); - writer.Write (assembly.DebugDataSize); - - writer.Write (assembly.ConfigDataOffset); - writer.Write (assembly.ConfigDataSize); - } - } - - void AdjustOffsets (AssemblyStoreIndexEntry assembly, uint offsetFixup) - { - if (offsetFixup == 0) { - return; - } - - assembly.DataOffset += offsetFixup; - - if (assembly.DebugDataOffset > 0) { - assembly.DebugDataOffset += offsetFixup; - } - - if (assembly.ConfigDataOffset > 0) { - assembly.ConfigDataOffset += offsetFixup; - } - } - - AssemblyStoreIndexEntry WriteAssembly (BinaryWriter writer, AssemblyStoreAssemblyInfo assembly, string assemblyName, uint localBlobIndex) - { - uint offset; - uint size; - - (offset, size) = WriteFile (assembly.FilesystemAssemblyPath, true); - - // NOTE: globalAssemblIndex++ is not thread safe but it **must** increase monotonically (see also ArchAssemblyStore.Generate for a special case) - var ret = new AssemblyStoreIndexEntry (assemblyName, ID, GlobalIndexCounter.Increment (), localBlobIndex) { - DataOffset = offset, - DataSize = size, - }; - - (offset, size) = WriteFile (assembly.DebugInfoPath, required: false); - if (offset != 0 && size != 0) { - ret.DebugDataOffset = offset; - ret.DebugDataSize = size; - } - - // Config files must end with \0 (nul) - (offset, size) = WriteFile (assembly.ConfigPath, required: false, appendNul: true); - if (offset != 0 && size != 0) { - ret.ConfigDataOffset = offset; - ret.ConfigDataSize = size; - } - - return ret; - - (uint offset, uint size) WriteFile (string filePath, bool required, bool appendNul = false) - { - if (!File.Exists (filePath)) { - if (required) { - throw new InvalidOperationException ($"Required file '{filePath}' not found"); - } - - return (0, 0); - } - - var fi = new FileInfo (filePath); - if (fi.Length == 0) { - return (0, 0); - } - - if (fi.Length > UInt32.MaxValue || writer.BaseStream.Position + fi.Length > UInt32.MaxValue) { - throw new InvalidOperationException ($"Writing assembly '{filePath}' to assembly blob would exceed the maximum allowed data size."); - } - - uint offset = (uint)writer.BaseStream.Position; - using (var fs = File.Open (filePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { - fs.CopyTo (writer.BaseStream); - } - - uint length = (uint)fi.Length; - if (appendNul) { - length++; - writer.Write ((byte)0); - } - - return (offset, length); - } - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs index c5c166fb787..ad4a04cf95a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs @@ -1,48 +1,53 @@ using System; using System.IO; -namespace Xamarin.Android.Tasks +using Microsoft.Build.Framework; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +class AssemblyStoreAssemblyInfo { - class AssemblyStoreAssemblyInfo + public AndroidTargetArch Arch { get; } + public FileInfo SourceFile { get; } + public string AssemblyName { get; } + public byte[] AssemblyNameBytes { get; } + public string AssemblyNameNoExt { get; } + public byte[] AssemblyNameNoExtBytes { get; } + public FileInfo? SymbolsFile { get; set; } + public FileInfo? ConfigFile { get; set; } + + public AssemblyStoreAssemblyInfo (string sourceFilePath, ITaskItem assembly) { - public string FilesystemAssemblyPath { get; } - public string ArchiveAssemblyPath { get; } - public string DebugInfoPath { get; private set; } - public string ConfigPath { get; private set; } - public string Abi { get; } - - public AssemblyStoreAssemblyInfo (string filesystemAssemblyPath, string archiveAssemblyPath, string abi) - { - if (String.IsNullOrEmpty (filesystemAssemblyPath)) { - throw new ArgumentException ("must not be null or empty", nameof (filesystemAssemblyPath)); - } + Arch = MonoAndroidHelper.GetTargetArch (assembly); + if (Arch == AndroidTargetArch.None) { + throw new InvalidOperationException ($"Internal error: assembly item '{assembly}' lacks ABI information metadata"); + } - if (String.IsNullOrEmpty (archiveAssemblyPath)) { - throw new ArgumentException ("must not be null or empty", nameof (archiveAssemblyPath)); - } + SourceFile = new FileInfo (sourceFilePath); - FilesystemAssemblyPath = filesystemAssemblyPath; - ArchiveAssemblyPath = archiveAssemblyPath; - Abi = abi; + string? name = Path.GetFileName (SourceFile.Name); + if (name == null) { + throw new InvalidOperationException ("Internal error: info without assembly name"); } - public void SetDebugInfoPath (string path) - { - DebugInfoPath = GetExistingPath (path); + if (name.EndsWith (".lz4", StringComparison.OrdinalIgnoreCase)) { + name = Path.GetFileNameWithoutExtension (name); } - public void SetConfigPath (string path) - { - ConfigPath = GetExistingPath (path); + string nameNoExt = Path.GetFileNameWithoutExtension (name); + string? culture = assembly.GetMetadata ("Culture"); + if (!String.IsNullOrEmpty (culture)) { + name = $"{culture}/{name}"; + nameNoExt = $"{culture}/{nameNoExt}"; } - string GetExistingPath (string path) - { - if (String.IsNullOrEmpty (path) || !File.Exists (path)) { - return String.Empty; - } + (AssemblyName, AssemblyNameBytes) = SetName (name); + (AssemblyNameNoExt, AssemblyNameNoExtBytes) = SetName (nameNoExt); - return path; + (string name, byte[] bytes) SetName (string assemblyName) + { + return (assemblyName, MonoAndroidHelper.Utf8StringToBytes (assemblyName)); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.Classes.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.Classes.cs new file mode 100644 index 00000000000..680d8966070 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.Classes.cs @@ -0,0 +1,65 @@ +namespace Xamarin.Android.Tasks; + +partial class AssemblyStoreGenerator +{ + sealed class AssemblyStoreHeader + { + public const uint NativeSize = 5 * sizeof (uint); + + public readonly uint magic = ASSEMBLY_STORE_MAGIC; + public readonly uint version; + public readonly uint entry_count; + public readonly uint index_entry_count; + + // Index size in bytes + public readonly uint index_size; + + public AssemblyStoreHeader (uint version, uint entry_count, uint index_entry_count, uint index_size) + { + this.version = version; + this.entry_count = entry_count; + this.index_entry_count = index_entry_count; + this.index_size = index_size; + } +#if XABT_TESTS + public AssemblyStoreHeader (uint magic, uint version, uint entry_count, uint index_entry_count, uint index_size) + : this (version, entry_count, index_entry_count, index_size) + { + this.magic = magic; + } +#endif + } + + sealed class AssemblyStoreIndexEntry + { + public const uint NativeSize32 = 2 * sizeof (uint); + public const uint NativeSize64 = sizeof (ulong) + sizeof (uint); + + public readonly string name; + public readonly ulong name_hash; + public readonly uint descriptor_index; + + public AssemblyStoreIndexEntry (string name, ulong name_hash, uint descriptor_index) + { + this.name = name; + this.name_hash = name_hash; + this.descriptor_index = descriptor_index; + } + } + + sealed class AssemblyStoreEntryDescriptor + { + public const uint NativeSize = 7 * sizeof (uint); + + public uint mapping_index; + + public uint data_offset; + public uint data_size; + + public uint debug_data_offset; + public uint debug_data_size; + + public uint config_data_offset; + public uint config_data_size; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs index d60af903bc9..010b607f0c7 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs @@ -1,133 +1,343 @@ using System; using System.Collections.Generic; using System.IO; -using System.Text; -using Microsoft.Build.Framework; +using Microsoft.Android.Build.Tasks; using Microsoft.Build.Utilities; +using Xamarin.Android.Tools; -namespace Xamarin.Android.Tasks +namespace Xamarin.Android.Tasks; + +// +// Assembly store format +// +// Each target ABI/architecture has a single assembly store file, composed of the following parts: +// +// [HEADER] +// [INDEX] +// [ASSEMBLY_DESCRIPTORS] +// [ASSEMBLY DATA] +// +// Formats of the sections above are as follows: +// +// HEADER (fixed size) +// [MAGIC] uint; value: 0x41424158 +// [FORMAT_VERSION] uint; store format version number +// [ENTRY_COUNT] uint; number of entries in the store +// [INDEX_ENTRY_COUNT] uint; number of entries in the index +// [INDEX_SIZE] uint; index size in bytes +// +// INDEX (variable size, HEADER.ENTRY_COUNT*2 entries, for assembly names with and without the extension) +// [NAME_HASH] uint on 32-bit platforms, ulong on 64-bit platforms; xxhash of the assembly name +// [DESCRIPTOR_INDEX] uint; index into in-store assembly descriptor array +// +// ASSEMBLY_DESCRIPTORS (variable size, HEADER.ENTRY_COUNT entries), each entry formatted as follows: +// [MAPPING_INDEX] uint; index into a runtime array where assembly data pointers are stored +// [DATA_OFFSET] uint; offset from the beginning of the store to the start of assembly data +// [DATA_SIZE] uint; size of the stored assembly data +// [DEBUG_DATA_OFFSET] uint; offset from the beginning of the store to the start of assembly PDB data, 0 if absent +// [DEBUG_DATA_SIZE] uint; size of the stored assembly PDB data, 0 if absent +// [CONFIG_DATA_OFFSET] uint; offset from the beginning of the store to the start of assembly .config contents, 0 if absent +// [CONFIG_DATA_SIZE] uint; size of the stored assembly .config contents, 0 if absent +// +// ASSEMBLY_NAMES (variable size, HEADER.ENTRY_COUNT entries), each entry formatted as follows: +// [NAME_LENGTH] uint: length of assembly name +// [NAME] byte: UTF-8 bytes of assembly name, without the NUL terminator +// +partial class AssemblyStoreGenerator { - class AssemblyStoreGenerator + // The two constants below must match their counterparts in src/monodroid/jni/xamarin-app.hh + const uint ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian, must match the BUNDLED_ASSEMBLIES_BLOB_MAGIC native constant + + // Bit 31 is set for 64-bit platforms, cleared for the 32-bit ones + const uint ASSEMBLY_STORE_FORMAT_VERSION_64BIT = 0x80000002; // Must match the ASSEMBLY_STORE_FORMAT_VERSION native constant + const uint ASSEMBLY_STORE_FORMAT_VERSION_32BIT = 0x00000002; + + const uint ASSEMBLY_STORE_ABI_AARCH64 = 0x00010000; + const uint ASSEMBLY_STORE_ABI_ARM = 0x00020000; + const uint ASSEMBLY_STORE_ABI_X64 = 0x00030000; + const uint ASSEMBLY_STORE_ABI_X86 = 0x00040000; + + readonly TaskLoggingHelper log; + readonly Dictionary> assemblies; + + public AssemblyStoreGenerator (TaskLoggingHelper log) { - sealed class Store - { - public AssemblyStore Common; - public AssemblyStore Arch; + this.log = log; + assemblies = new Dictionary> (); + } + + public void Add (AssemblyStoreAssemblyInfo asmInfo) + { + if (!assemblies.TryGetValue (asmInfo.Arch, out List infos)) { + infos = new List (); + assemblies.Add (asmInfo.Arch, infos); } - readonly string archiveAssembliesPrefix; - readonly TaskLoggingHelper log; + infos.Add (asmInfo); + } - // NOTE: when/if we have parallel BuildApk these should become concurrent collections - readonly Dictionary stores = new Dictionary (StringComparer.Ordinal); + public Dictionary Generate (string baseOutputDirectory) + { + var ret = new Dictionary (); - AssemblyStore indexStore; + foreach (var kvp in assemblies) { + string storePath = Generate (baseOutputDirectory, kvp.Key, kvp.Value); + ret.Add (kvp.Key, storePath); + } - // IDs must be counted per AssemblyStoreGenerator instance because it's possible that a single build will create more than one instance of the class and each time - // the stores must be assigned IDs starting from 0, or there will be errors due to "missing" index store - readonly Dictionary apkIds = new Dictionary (StringComparer.Ordinal); + return ret; + } - // Global assembly index must be restarted from 0 for the same reasons as apkIds above and at the same time it must be unique for each assembly added to **any** - // assembly store, thus we need to keep the state here - AssemblyStoreGlobalIndex globalIndexCounter = new AssemblyStoreGlobalIndex (); + string Generate (string baseOutputDirectory, AndroidTargetArch arch, List infos) + { + (bool is64Bit, uint abiFlag) = arch switch { + AndroidTargetArch.Arm => (false, ASSEMBLY_STORE_ABI_ARM), + AndroidTargetArch.X86 => (false, ASSEMBLY_STORE_ABI_X86), + AndroidTargetArch.Arm64 => (true, ASSEMBLY_STORE_ABI_AARCH64), + AndroidTargetArch.X86_64 => (true, ASSEMBLY_STORE_ABI_X64), + _ => throw new NotSupportedException ($"Internal error: arch {arch} not supported") + }; - public AssemblyStoreGenerator (string archiveAssembliesPrefix, TaskLoggingHelper log) - { - if (String.IsNullOrEmpty (archiveAssembliesPrefix)) { - throw new ArgumentException ("must not be null or empty", nameof (archiveAssembliesPrefix)); - } + string androidAbi = MonoAndroidHelper.ArchToAbi (arch); + uint infoCount = (uint)infos.Count; + string storePath = Path.Combine (baseOutputDirectory, androidAbi, $"assemblies.{androidAbi}.blob.so"); + var index = new List (); + var descriptors = new List (); + ulong namesSize = 0; - this.archiveAssembliesPrefix = archiveAssembliesPrefix; - this.log = log; + foreach (AssemblyStoreAssemblyInfo info in infos) { + namesSize += (ulong)info.AssemblyNameBytes.Length; + namesSize += sizeof (uint); } - public void Add (string apkName, AssemblyStoreAssemblyInfo storeAssembly) - { - if (String.IsNullOrEmpty (apkName)) { - throw new ArgumentException ("must not be null or empty", nameof (apkName)); - } + ulong assemblyDataStart = (infoCount * IndexEntrySize () * 2) + (AssemblyStoreEntryDescriptor.NativeSize * infoCount) + AssemblyStoreHeader.NativeSize + namesSize; + // We'll start writing to the stream after we seek to the position just after the header, index, descriptors and name data. + ulong curPos = assemblyDataStart; - Store store; - if (!stores.ContainsKey (apkName)) { - store = new Store { - Common = new CommonAssemblyStore (apkName, archiveAssembliesPrefix, log, GetNextStoreID (apkName), globalIndexCounter), - Arch = new ArchAssemblyStore (apkName, archiveAssembliesPrefix, log, GetNextStoreID (apkName), globalIndexCounter) - }; + using var fs = File.Open (storePath, FileMode.Create, FileAccess.Write, FileShare.Read); + fs.Seek ((long)curPos, SeekOrigin.Begin); - stores.Add (apkName, store); - SetIndexStore (store.Common); - SetIndexStore (store.Arch); - } + foreach (AssemblyStoreAssemblyInfo info in infos) { + (AssemblyStoreEntryDescriptor desc, curPos) = MakeDescriptor (info, curPos); + desc.mapping_index = (uint)descriptors.Count; + descriptors.Add (desc); - store = stores[apkName]; - if (String.IsNullOrEmpty (storeAssembly.Abi)) { - store.Common.Add (storeAssembly); - } else { - store.Arch.Add (storeAssembly); + if ((uint)fs.Position != desc.data_offset) { + throw new InvalidOperationException ($"Internal error: corrupted store '{storePath}' stream"); } - void SetIndexStore (AssemblyStore b) - { - if (!b.IsIndexStore) { - return; - } + ulong name_with_ext_hash = MonoAndroidHelper.GetXxHash (info.AssemblyNameBytes, is64Bit); + ulong name_no_ext_hash = MonoAndroidHelper.GetXxHash (info.AssemblyNameNoExtBytes, is64Bit); + index.Add (new AssemblyStoreIndexEntry (info.AssemblyName, name_with_ext_hash, desc.mapping_index)); + index.Add (new AssemblyStoreIndexEntry (info.AssemblyNameNoExt, name_no_ext_hash, desc.mapping_index)); + + CopyData (info.SourceFile, fs, storePath); + CopyData (info.SymbolsFile, fs, storePath); + CopyData (info.ConfigFile, fs, storePath); + } + fs.Flush (); + fs.Seek (0, SeekOrigin.Begin); - if (indexStore != null) { - throw new InvalidOperationException ("Index store already set!"); - } + uint storeVersion = is64Bit ? ASSEMBLY_STORE_FORMAT_VERSION_64BIT : ASSEMBLY_STORE_FORMAT_VERSION_32BIT; + var header = new AssemblyStoreHeader (storeVersion | abiFlag, infoCount, (uint)index.Count, (uint)(index.Count * IndexEntrySize ())); + using var writer = new BinaryWriter (fs); + WriteHeader (writer, header); - indexStore = b; - } + using var manifestFs = File.Open ($"{storePath}.manifest", FileMode.Create, FileAccess.Write, FileShare.Read); + using var mw = new StreamWriter (manifestFs, new System.Text.UTF8Encoding (false)); + WriteIndex (writer, mw, index, descriptors, is64Bit); + mw.Flush (); + + Console.WriteLine ($"Number of descriptors: {descriptors.Count}; index entries: {index.Count}"); + Console.WriteLine ($"Header size: {AssemblyStoreHeader.NativeSize}; index entry size: {IndexEntrySize ()}; descriptor size: {AssemblyStoreEntryDescriptor.NativeSize}"); + + WriteDescriptors (writer, descriptors); + WriteNames (writer, infos); + writer.Flush (); + + if (fs.Position != (long)assemblyDataStart) { + Console.WriteLine ($"fs.Position == {fs.Position}; assemblyDataStart == {assemblyDataStart}"); + throw new InvalidOperationException ($"Internal error: store '{storePath}' position is different than metadata size after header write"); } - uint GetNextStoreID (string apkName) - { - // NOTE: NOT thread safe, if we ever have parallel runs of BuildApk this operation must either be atomic or protected with a lock - if (!apkIds.ContainsKey (apkName)) { - apkIds.Add (apkName, 0); - } - return apkIds[apkName]++; + return storePath; + + uint IndexEntrySize () => is64Bit ? AssemblyStoreIndexEntry.NativeSize64 : AssemblyStoreIndexEntry.NativeSize32; + } + + void CopyData (FileInfo? src, Stream dest, string storePath) + { + if (src == null) { + return; + } + + log.LogDebugMessage ($"Adding file '{src.Name}' to assembly store '{storePath}'"); + using var fs = src.Open (FileMode.Open, FileAccess.Read, FileShare.Read); + fs.CopyTo (dest); + } + + static (AssemblyStoreEntryDescriptor desc, ulong newPos) MakeDescriptor (AssemblyStoreAssemblyInfo info, ulong curPos) + { + var ret = new AssemblyStoreEntryDescriptor { + data_offset = (uint)curPos, + data_size = GetDataLength (info.SourceFile), + }; + if (info.SymbolsFile != null) { + ret.debug_data_offset = ret.data_offset + ret.data_size; + ret.debug_data_size = GetDataLength (info.SymbolsFile); + } + + if (info.ConfigFile != null) { + ret.config_data_offset = ret.data_offset + ret.data_size + ret.debug_data_size; + ret.config_data_size = GetDataLength (info.ConfigFile); + } + + curPos += ret.data_size + ret.debug_data_size + ret.config_data_size; + if (curPos > UInt32.MaxValue) { + throw new NotSupportedException ("Assembly store size exceeds the maximum supported value"); } - public Dictionary> Generate (string outputDirectory) - { - if (stores.Count == 0) { - return null; + return (ret, curPos); + + uint GetDataLength (FileInfo? info) { + if (info == null) { + return 0; } - if (indexStore == null) { - throw new InvalidOperationException ("Index store not found"); + if (info.Length > UInt32.MaxValue) { + throw new NotSupportedException ($"File '{info.Name}' exceeds the maximum supported size"); } - var globalIndex = new List (); - var ret = new Dictionary> (StringComparer.Ordinal); - string indexStoreApkName = null; - foreach (var kvp in stores) { - string apkName = kvp.Key; - Store store = kvp.Value; + return (uint)info.Length; + } + } - if (!ret.ContainsKey (apkName)) { - ret.Add (apkName, new List ()); - } + void WriteHeader (BinaryWriter writer, AssemblyStoreHeader header) + { + writer.Write (header.magic); + writer.Write (header.version); + writer.Write (header.entry_count); + writer.Write (header.index_entry_count); + writer.Write (header.index_size); + } +#if XABT_TESTS + AssemblyStoreHeader ReadHeader (BinaryReader reader) + { + reader.BaseStream.Seek (0, SeekOrigin.Begin); + uint magic = reader.ReadUInt32 (); + uint version = reader.ReadUInt32 (); + uint entry_count = reader.ReadUInt32 (); + uint index_entry_count = reader.ReadUInt32 (); + uint index_size = reader.ReadUInt32 (); + + return new AssemblyStoreHeader (magic, version, entry_count, index_entry_count, index_size); + } +#endif - if (store.Common == indexStore || store.Arch == indexStore) { - indexStoreApkName = apkName; - } + void WriteIndex (BinaryWriter writer, StreamWriter manifestWriter, List index, List descriptors, bool is64Bit) + { + index.Sort ((AssemblyStoreIndexEntry a, AssemblyStoreIndexEntry b) => a.name_hash.CompareTo (b.name_hash)); - GenerateStore (store.Common, apkName); - GenerateStore (store.Arch, apkName); + foreach (AssemblyStoreIndexEntry entry in index) { + if (is64Bit) { + writer.Write (entry.name_hash); + manifestWriter.Write ($"0x{entry.name_hash:x}"); + } else { + writer.Write ((uint)entry.name_hash); + manifestWriter.Write ($"0x{(uint)entry.name_hash:x}"); } + writer.Write (entry.descriptor_index); + manifestWriter.Write ($" di:{entry.descriptor_index}"); - string manifestPath = indexStore.WriteIndex (globalIndex); - ret[indexStoreApkName].Add (manifestPath); + AssemblyStoreEntryDescriptor desc = descriptors[(int)entry.descriptor_index]; + manifestWriter.Write ($" mi:{desc.mapping_index}"); + manifestWriter.Write ($" do:{desc.data_offset}"); + manifestWriter.Write ($" ds:{desc.data_size}"); + manifestWriter.Write ($" ddo:{desc.debug_data_offset}"); + manifestWriter.Write ($" dds:{desc.debug_data_size}"); + manifestWriter.Write ($" cdo:{desc.config_data_offset}"); + manifestWriter.Write ($" cds:{desc.config_data_size}"); + manifestWriter.WriteLine ($" {entry.name}"); + } + } - return ret; + List ReadIndex (BinaryReader reader, AssemblyStoreHeader header) + { + if (header.index_entry_count > Int32.MaxValue) { + throw new InvalidOperationException ("Assembly store index is too big"); + } + + var index = new List ((int)header.index_entry_count); + reader.BaseStream.Seek (AssemblyStoreHeader.NativeSize, SeekOrigin.Begin); - void GenerateStore (AssemblyStore store, string apkName) - { - store.Generate (outputDirectory, globalIndex, ret[apkName]); + bool is64Bit = (header.version & ASSEMBLY_STORE_FORMAT_VERSION_64BIT) == ASSEMBLY_STORE_FORMAT_VERSION_64BIT; + for (int i = 0; i < (int)header.index_entry_count; i++) { + ulong name_hash; + if (is64Bit) { + name_hash = reader.ReadUInt64 (); + } else { + name_hash = reader.ReadUInt32 (); } + + uint descriptor_index = reader.ReadUInt32 (); + index.Add (new AssemblyStoreIndexEntry (String.Empty, name_hash, descriptor_index)); + } + + return index; + } + + void WriteDescriptors (BinaryWriter writer, List descriptors) + { + foreach (AssemblyStoreEntryDescriptor desc in descriptors) { + writer.Write (desc.mapping_index); + writer.Write (desc.data_offset); + writer.Write (desc.data_size); + writer.Write (desc.debug_data_offset); + writer.Write (desc.debug_data_size); + writer.Write (desc.config_data_offset); + writer.Write (desc.config_data_size); + } + } + + List ReadDescriptors (BinaryReader reader, AssemblyStoreHeader header) + { + if (header.entry_count > Int32.MaxValue) { + throw new InvalidOperationException ("Assembly store descriptor table is too big"); + } + + var descriptors = new List (); + reader.BaseStream.Seek (AssemblyStoreHeader.NativeSize + header.index_size, SeekOrigin.Begin); + + for (int i = 0; i < (int)header.entry_count; i++) { + uint mapping_index = reader.ReadUInt32 (); + uint data_offset = reader.ReadUInt32 (); + uint data_size = reader.ReadUInt32 (); + uint debug_data_offset = reader.ReadUInt32 (); + uint debug_data_size = reader.ReadUInt32 (); + uint config_data_offset = reader.ReadUInt32 (); + uint config_data_size = reader.ReadUInt32 (); + + var desc = new AssemblyStoreEntryDescriptor { + mapping_index = mapping_index, + data_offset = data_offset, + data_size = data_size, + debug_data_offset = debug_data_offset, + debug_data_size = debug_data_size, + config_data_offset = config_data_offset, + config_data_size = config_data_size, + }; + descriptors.Add (desc); + } + + return descriptors; + } + + void WriteNames (BinaryWriter writer, List infos) + { + foreach (AssemblyStoreAssemblyInfo info in infos) { + writer.Write ((uint)info.AssemblyNameBytes.Length); + writer.Write (info.AssemblyNameBytes); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGlobalIndex.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGlobalIndex.cs deleted file mode 100644 index 6ce93f11f9d..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGlobalIndex.cs +++ /dev/null @@ -1,29 +0,0 @@ -namespace Xamarin.Android.Tasks -{ - // This class may seem weird, but it's designed with the specific needs of AssemblyStore instances in mind and also prepared for thread-safe use in the future, should the - // need arise - sealed class AssemblyStoreGlobalIndex - { - uint value = 0; - - public uint Value => value; - - /// - /// Increments the counter and returns its previous value - /// - public uint Increment () - { - uint ret = value++; - return ret; - } - - public void Subtract (uint count) - { - if (value < count) { - return; - } - - value -= count; - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreIndexEntry.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreIndexEntry.cs deleted file mode 100644 index 51d2bd8ca77..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreIndexEntry.cs +++ /dev/null @@ -1,43 +0,0 @@ -using System; -using System.IO.Hashing; -using System.Text; - -namespace Xamarin.Android.Tasks -{ - class AssemblyStoreIndexEntry - { - public string Name { get; } - public uint StoreID { get; } - public uint MappingIndex { get; } - public uint LocalBlobIndex { get; } - - // Hash values must have the same type as they are inside a union in the native code - public ulong NameHash64 { get; } - public ulong NameHash32 { get; } - - public uint DataOffset { get; set; } - public uint DataSize { get; set; } - - public uint DebugDataOffset { get; set; } - public uint DebugDataSize { get; set; } - - public uint ConfigDataOffset { get; set; } - public uint ConfigDataSize { get; set; } - - public AssemblyStoreIndexEntry (string name, uint blobID, uint mappingIndex, uint localBlobIndex) - { - if (String.IsNullOrEmpty (name)) { - throw new ArgumentException ("must not be null or empty", nameof (name)); - } - - Name = name; - StoreID = blobID; - MappingIndex = mappingIndex; - LocalBlobIndex = localBlobIndex; - - byte[] nameBytes = Encoding.UTF8.GetBytes (name); - NameHash32 = XxHash32.HashToUInt32 (nameBytes); - NameHash64 = XxHash64.HashToUInt64 (nameBytes); - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs deleted file mode 100644 index 9709a200e5a..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs +++ /dev/null @@ -1,39 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Text; - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -namespace Xamarin.Android.Tasks -{ - class CommonAssemblyStore : AssemblyStore - { - readonly List assemblies; - - public CommonAssemblyStore (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyStoreGlobalIndex globalIndexCounter) - : base (apkName, archiveAssembliesPrefix, log, id, globalIndexCounter) - { - assemblies = new List (); - } - - public override void Add (AssemblyStoreAssemblyInfo blobAssembly) - { - if (!String.IsNullOrEmpty (blobAssembly.Abi)) { - throw new InvalidOperationException ($"Architecture-specific assembly cannot be added to an architecture-agnostic blob ({blobAssembly.FilesystemAssemblyPath})"); - } - - assemblies.Add (blobAssembly); - } - - public override void Generate (string outputDirectory, List globalIndex, List blobPaths) - { - if (assemblies.Count == 0) { - return; - } - - Generate (Path.Combine (outputDirectory, $"{ApkName}_{BlobPrefix}{BlobExtension}"), assemblies, globalIndex, blobPaths); - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssembliesNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssembliesNativeAssemblyGenerator.cs index 6cb7f251f8a..29d7fc1665b 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssembliesNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssembliesNativeAssemblyGenerator.cs @@ -4,6 +4,7 @@ using Microsoft.Build.Utilities; using Xamarin.Android.Tasks.LLVMIR; +using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks { @@ -30,9 +31,15 @@ sealed class CompressedAssemblyDescriptorContextDataProvider : NativeAssemblerSt [NativeAssemblerStructContextDataProvider (typeof (CompressedAssemblyDescriptorContextDataProvider))] sealed class CompressedAssemblyDescriptor { + [NativeAssembler (Ignore = true)] + public uint Index; + [NativeAssembler (Ignore = true)] public string BufferSymbolName; + [NativeAssembler (Ignore = true)] + public string AssemblyName; + public uint uncompressed_file_size; public bool loaded; @@ -64,65 +71,93 @@ sealed class CompressedAssemblies public CompressedAssemblyDescriptor descriptors; }; - IDictionary assemblies; + IDictionary>? archAssemblies; StructureInfo compressedAssemblyDescriptorStructureInfo; StructureInfo compressedAssembliesStructureInfo; + Dictionary>> archData = new Dictionary>> (); - public CompressedAssembliesNativeAssemblyGenerator (TaskLoggingHelper log, IDictionary assemblies) + public CompressedAssembliesNativeAssemblyGenerator (TaskLoggingHelper log, IDictionary>? archAssemblies) : base (log) { - this.assemblies = assemblies; + this.archAssemblies = archAssemblies; } - void InitCompressedAssemblies (out List>? compressedAssemblyDescriptors, - out StructureInstance? compressedAssemblies, + void InitCompressedAssemblies (out List? compressedAssemblies, + out List? compressedAssemblyDescriptors, out List? buffers) { - if (assemblies == null || assemblies.Count == 0) { - compressedAssemblyDescriptors = null; + if (archAssemblies == null || archAssemblies.Count == 0) { compressedAssemblies = null; + compressedAssemblyDescriptors = null; buffers = null; return; } - ulong counter = 0; - compressedAssemblyDescriptors = new List> (assemblies.Count); - buffers = new List (assemblies.Count); - foreach (var kvp in assemblies) { - string assemblyName = kvp.Key; - CompressedAssemblyInfo info = kvp.Value; - - string bufferName = $"__compressedAssemblyData_{counter++}"; - var descriptor = new CompressedAssemblyDescriptor { - BufferSymbolName = bufferName, - uncompressed_file_size = info.FileSize, - loaded = false, - data = 0 - }; + buffers = new List (); + foreach (var kvpArch in archAssemblies) { + foreach (var kvp in kvpArch.Value) { + CompressedAssemblyInfo info = kvp.Value; + + if (!archData.TryGetValue (info.TargetArch, out List> descriptors)) { + descriptors = new List> (); + archData.Add (info.TargetArch, descriptors); + } + + string bufferName = $"__compressedAssemblyData_{info.DescriptorIndex}"; + var descriptor = new CompressedAssemblyDescriptor { + Index = info.DescriptorIndex, + BufferSymbolName = bufferName, + AssemblyName = info.AssemblyName, + uncompressed_file_size = info.FileSize, + loaded = false, + data = 0 + }; + + descriptors.Add (new StructureInstance (compressedAssemblyDescriptorStructureInfo, descriptor)); + + var buffer = new LlvmIrGlobalVariable (typeof(List), descriptor.BufferSymbolName, LlvmIrVariableOptions.LocalWritable) { + ArrayItemCount = descriptor.uncompressed_file_size, + TargetArch = info.TargetArch, + ZeroInitializeArray = true, + }; + buffers.Add (buffer); + } + } - var bufferVar = new LlvmIrGlobalVariable (typeof(List), bufferName, LlvmIrVariableOptions.LocalWritable) { - ZeroInitializeArray = true, - ArrayItemCount = descriptor.uncompressed_file_size, + compressedAssemblies = new List (); + compressedAssemblyDescriptors = new List (); + foreach (var kvp in archData) { + List> descriptors = kvp.Value; + descriptors.Sort ((StructureInstance a, StructureInstance b) => a.Instance.Index.CompareTo (b.Instance.Index)); + + var variable = new LlvmIrGlobalVariable (typeof(StructureInstance), CompressedAssembliesSymbolName) { + Options = LlvmIrVariableOptions.GlobalWritable, + TargetArch = kvp.Key, + Value = new StructureInstance (compressedAssembliesStructureInfo, new CompressedAssemblies { count = (uint)descriptors.Count, }), }; - buffers.Add (bufferVar); + compressedAssemblies.Add (variable); - compressedAssemblyDescriptors.Add (new StructureInstance (compressedAssemblyDescriptorStructureInfo, descriptor)); + variable = new LlvmIrGlobalVariable (typeof(List>), DescriptorsArraySymbolName) { + GetArrayItemCommentCallback = GetCompressedAssemblyDescriptorsItemComment, + Options = LlvmIrVariableOptions.LocalWritable, + TargetArch = kvp.Key, + Value = descriptors, + }; + compressedAssemblyDescriptors.Add (variable); } - - compressedAssemblies = new StructureInstance (compressedAssembliesStructureInfo, new CompressedAssemblies { count = (uint)assemblies.Count }); } protected override void Construct (LlvmIrModule module) { MapStructures (module); - List>? compressedAssemblyDescriptors; - StructureInstance? compressedAssemblies; - List? buffers; + InitCompressedAssemblies ( + out List? compressedAssemblies, + out List? compressedAssemblyDescriptors, + out List? buffers + ); - InitCompressedAssemblies (out compressedAssemblyDescriptors, out compressedAssemblies, out buffers); - - if (compressedAssemblyDescriptors == null) { + if (archData.Count == 0) { module.AddGlobalVariable ( typeof(StructureInstance), CompressedAssembliesSymbolName, @@ -132,14 +167,34 @@ protected override void Construct (LlvmIrModule module) return; } - module.AddGlobalVariable (CompressedAssembliesSymbolName, compressedAssemblies, LlvmIrVariableOptions.GlobalWritable); - module.AddGlobalVariable (DescriptorsArraySymbolName, compressedAssemblyDescriptors, LlvmIrVariableOptions.LocalWritable); + module.Add (compressedAssemblies); + module.Add (compressedAssemblyDescriptors); module.Add (new LlvmIrGroupDelimiterVariable ()); module.Add (buffers); module.Add (new LlvmIrGroupDelimiterVariable ()); } + string? GetCompressedAssemblyDescriptorsItemComment (LlvmIrVariable v, LlvmIrModuleTarget target, ulong index, object? value, object? callerState) + { + List> descriptors = GetArchDescriptors (target); + if ((int)index >= descriptors.Count) { + throw new InvalidOperationException ($"Internal error: index {index} is too big for variable '{v.Name}'"); + } + StructureInstance desc = descriptors[(int)index]; + + return $" {index}: {desc.Instance.AssemblyName}"; + } + + List> GetArchDescriptors (LlvmIrModuleTarget target) + { + if (!archData.TryGetValue (target.TargetArch, out List> descriptors)) { + throw new InvalidOperationException ($"Internal error: missing compressed descriptors data for architecture '{target.TargetArch}'"); + } + + return descriptors; + } + void MapStructures (LlvmIrModule module) { compressedAssemblyDescriptorStructureInfo = module.MapStructure (); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs index b3f775a96b7..a5aa5f0dcd8 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs @@ -2,19 +2,25 @@ using System.IO; using Microsoft.Build.Framework; +using Xamarin.Android.Tools; + namespace Xamarin.Android.Tasks { class CompressedAssemblyInfo { const string CompressedAssembliesInfoKey = "__CompressedAssembliesInfo"; - public uint FileSize { get; } - public uint DescriptorIndex { get; set; } + public uint FileSize { get; } + public uint DescriptorIndex { get; } + public AndroidTargetArch TargetArch { get; } + public string AssemblyName { get; } - public CompressedAssemblyInfo (uint fileSize) + public CompressedAssemblyInfo (uint fileSize, uint descriptorIndex, AndroidTargetArch targetArch, string assemblyName) { FileSize = fileSize; - DescriptorIndex = 0; + DescriptorIndex = descriptorIndex; + TargetArch = targetArch; + AssemblyName = assemblyName; } public static string GetKey (string projectFullPath) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/IAssemblyResolverExtensions.cs b/src/Xamarin.Android.Build.Tasks/Utilities/IAssemblyResolverExtensions.cs new file mode 100644 index 00000000000..e35950739a8 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/IAssemblyResolverExtensions.cs @@ -0,0 +1,11 @@ +using Mono.Cecil; + +namespace Xamarin.Android.Tasks; + +static class AssebmlyResolverExtensions +{ + public static AssemblyDefinition? Resolve (this IAssemblyResolver resolver, string fullName, ReaderParameters? parameters = null) + { + return resolver?.Resolve (AssemblyNameReference.Parse (fullName), parameters); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs new file mode 100644 index 00000000000..d1a400bab20 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs @@ -0,0 +1,438 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; + +using Java.Interop.Tools.Cecil; +using Java.Interop.Tools.Diagnostics; +using Java.Interop.Tools.JavaCallableWrappers.Adapters; +using Java.Interop.Tools.JavaCallableWrappers.CallableWrapperMembers; +using Java.Interop.Tools.JavaCallableWrappers; +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Mono.Cecil; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +class JCWGeneratorContext +{ + public bool UseMarshalMethods { get; } + public AndroidTargetArch Arch { get; } + public TypeDefinitionCache TypeDefinitionCache { get; } + public XAAssemblyResolver Resolver { get; } + public IList JavaTypes { get; } + public ICollection ResolvedAssemblies { get; } + + public JCWGeneratorContext (AndroidTargetArch arch, XAAssemblyResolver res, ICollection resolvedAssemblies, List javaTypesForJCW, TypeDefinitionCache tdCache, bool useMarshalMethods) + { + Arch = arch; + Resolver = res; + ResolvedAssemblies = resolvedAssemblies; + JavaTypes = javaTypesForJCW.AsReadOnly (); + TypeDefinitionCache = tdCache; + UseMarshalMethods = useMarshalMethods; + } +} + +class JCWGenerator +{ + readonly TaskLoggingHelper log; + readonly JCWGeneratorContext context; + + public MarshalMethodsClassifier? Classifier { get; private set; } + + public JCWGenerator (TaskLoggingHelper log, JCWGeneratorContext context) + { + this.log = log; + this.context = context; + } + + /// + /// Performs marshal method classification, if marshal methods are used, but does not generate any code. + /// If marshal methods are used, this method will set the property to a valid + /// classifier instance on return. If marshal methods are disabled, this call is a no-op but it will + /// return true. + /// + public bool Classify (string androidSdkPlatform) + { + if (!context.UseMarshalMethods) { + return true; + } + + Classifier = MakeClassifier (); + return ProcessTypes ( + generateCode: false, + androidSdkPlatform, + Classifier, + outputPath: null, + applicationJavaClass: null + ); + } + + public bool GenerateAndClassify (string androidSdkPlatform, string outputPath, string applicationJavaClass) + { + if (context.UseMarshalMethods) { + Classifier = MakeClassifier (); + } + + return ProcessTypes ( + generateCode: true, + androidSdkPlatform, + Classifier, + outputPath, + applicationJavaClass + ); + } + + MarshalMethodsClassifier MakeClassifier () => new MarshalMethodsClassifier (context.Arch, context.TypeDefinitionCache, context.Resolver, log); + + bool ProcessTypes (bool generateCode, string androidSdkPlatform, MarshalMethodsClassifier? classifier, string? outputPath, string? applicationJavaClass) + { + if (generateCode && String.IsNullOrEmpty (outputPath)) { + throw new ArgumentException ("must not be null or empty", nameof (outputPath)); + } + + string monoInit = GetMonoInitSource (androidSdkPlatform); + bool hasExportReference = context.ResolvedAssemblies.Any (assembly => Path.GetFileName (assembly.ItemSpec) == "Mono.Android.Export.dll"); + bool ok = true; + + foreach (TypeDefinition type in context.JavaTypes) { + if (type.IsInterface) { + // Interfaces are in typemap but they shouldn't have JCW generated for them + continue; + } + + CallableWrapperType generator = CreateGenerator (type, classifier, monoInit, hasExportReference, applicationJavaClass); + if (!generateCode) { + continue; + } + + if (!GenerateCode (generator, type, outputPath, hasExportReference, classifier)) { + ok = false; + } + } + + return ok; + } + + bool GenerateCode (CallableWrapperType generator, TypeDefinition type, string outputPath, bool hasExportReference, MarshalMethodsClassifier? classifier) + { + bool ok = true; + using var writer = MemoryStreamPool.Shared.CreateStreamWriter (); + var writer_options = new CallableWrapperWriterOptions { + CodeGenerationTarget = JavaPeerStyle.XAJavaInterop1 + }; + + try { + generator.Generate (writer, writer_options); + if (context.UseMarshalMethods) { + if (classifier.FoundDynamicallyRegisteredMethods (type)) { + log.LogWarning ($"Type '{type.GetAssemblyQualifiedName (context.TypeDefinitionCache)}' will register some of its Java override methods dynamically. This may adversely affect runtime performance. See preceding warnings for names of dynamically registered methods."); + } + } + writer.Flush (); + + string path = generator.GetDestinationPath (outputPath); + Files.CopyIfStreamChanged (writer.BaseStream, path); + if (generator.HasExport && !hasExportReference) { + Diagnostic.Error (4210, Properties.Resources.XA4210); + } + } catch (XamarinAndroidException xae) { + ok = false; + log.LogError ( + subcategory: "", + errorCode: "XA" + xae.Code, + helpKeyword: string.Empty, + file: xae.SourceFile, + lineNumber: xae.SourceLine, + columnNumber: 0, + endLineNumber: 0, + endColumnNumber: 0, + message: xae.MessageWithoutCode, + messageArgs: Array.Empty () + ); + } catch (DirectoryNotFoundException ex) { + ok = false; + if (OS.IsWindows) { + Diagnostic.Error (5301, Properties.Resources.XA5301, type.FullName, ex); + } else { + Diagnostic.Error (4209, Properties.Resources.XA4209, type.FullName, ex); + } + } catch (Exception ex) { + ok = false; + Diagnostic.Error (4209, Properties.Resources.XA4209, type.FullName, ex); + } + + return ok; + } + + CallableWrapperType CreateGenerator (TypeDefinition type, MarshalMethodsClassifier? classifier, string monoInit, bool hasExportReference, string? applicationJavaClass) + { + var reader_options = new CallableWrapperReaderOptions { + DefaultApplicationJavaClass = applicationJavaClass, + DefaultGenerateOnCreateOverrides = false, // this was used only when targetting Android API <= 10, which is no longer supported + DefaultMonoRuntimeInitialization = monoInit, + MethodClassifier = classifier, + }; + + return CecilImporter.CreateType (type, context.TypeDefinitionCache, reader_options); + } + + static string GetMonoInitSource (string androidSdkPlatform) + { + if (String.IsNullOrEmpty (androidSdkPlatform)) { + throw new ArgumentException ("must not be null or empty", nameof (androidSdkPlatform)); + } + + // Lookup the mono init section from MonoRuntimeProvider: + // Mono Runtime Initialization {{{ + // }}} + var builder = new StringBuilder (); + var runtime = "Bundled"; + var api = ""; + if (int.TryParse (androidSdkPlatform, out int apiLevel) && apiLevel < 21) { + api = ".20"; + } + + var assembly = Assembly.GetExecutingAssembly (); + using var s = assembly.GetManifestResourceStream ($"MonoRuntimeProvider.{runtime}{api}.java"); + using var reader = new StreamReader (s); + bool copy = false; + string? line; + while ((line = reader.ReadLine ()) != null) { + if (string.CompareOrdinal ("\t\t// Mono Runtime Initialization {{{", line) == 0) { + copy = true; + } + + if (copy) { + builder.AppendLine (line); + } + + if (string.CompareOrdinal ("\t\t// }}}", line) == 0) { + break; + } + } + + return builder.ToString (); + } + + public static void EnsureAllArchitecturesAreIdentical (TaskLoggingHelper logger, Dictionary javaStubStates) + { + if (javaStubStates.Count <= 1) { + return; + } + + // An expensive process, but we must be sure that all the architectures have the same data + NativeCodeGenState? templateState = null; + foreach (var kvp in javaStubStates) { + NativeCodeGenState state = kvp.Value; + + if (templateState == null) { + templateState = state; + continue; + } + + EnsureIdenticalCollections (logger, templateState, state); + EnsureClassifiersMatch (logger, templateState, state); + } + } + + static void EnsureIdenticalCollections (TaskLoggingHelper logger, NativeCodeGenState templateState, NativeCodeGenState state) + { + logger.LogDebugMessage ($"Ensuring Java type collection in architecture '{state.TargetArch}' matches the one in architecture '{templateState.TargetArch}'"); + + List templateTypes = templateState.AllJavaTypes; + List types = state.AllJavaTypes; + + if (types.Count != templateTypes.Count) { + throw new InvalidOperationException ($"Internal error: architecture '{state.TargetArch}' has a different number of types ({types.Count}) than the template architecture '{templateState.TargetArch}' ({templateTypes.Count})"); + } + + var matchedTemplateTypes = new HashSet (); + var mismatchedTypes = new List (); + + foreach (TypeDefinition type in types) { + TypeDefinition? matchedType = null; + + foreach (TypeDefinition templateType in templateTypes) { + if (matchedTemplateTypes.Contains (templateType) || !CheckWhetherTypesMatch (templateType, type)) { + continue; + } + + matchedTemplateTypes.Add (templateType); + matchedType = templateType; + break; + } + + if (matchedType == null) { + mismatchedTypes.Add (type); + } + } + + if (mismatchedTypes.Count > 0) { + logger.LogError ($"Architecture '{state.TargetArch}' has Java types which have no counterparts in template architecture '{templateState.TargetArch}':"); + foreach (TypeDefinition td in mismatchedTypes) { + logger.LogError ($" {td.FullName}"); + } + } + } + + static bool CheckWhetherTypesMatch (TypeDefinition templateType, TypeDefinition type) + { + // TODO: should we compare individual methods, fields, properties? + return String.Compare (templateType.FullName, type.FullName, StringComparison.Ordinal) == 0; + } + + static void EnsureClassifiersMatch (TaskLoggingHelper logger, NativeCodeGenState templateState, NativeCodeGenState state) + { + logger.LogDebugMessage ($"Ensuring marshal method classifier in architecture '{state.TargetArch}' matches the one in architecture '{templateState.TargetArch}'"); + + MarshalMethodsClassifier? templateClassifier = templateState.Classifier; + MarshalMethodsClassifier? classifier = state.Classifier; + + if (templateClassifier == null) { + if (classifier != null) { + throw new InvalidOperationException ($"Internal error: architecture '{templateState.TargetArch}' DOES NOT have a marshal methods classifier, unlike architecture '{state.TargetArch}'"); + } + return; + } + + if (classifier == null) { + throw new InvalidOperationException ($"Internal error: architecture '{templateState.TargetArch}' DOES have a marshal methods classifier, unlike architecture '{state.TargetArch}'"); + } + + if (templateClassifier.MarshalMethods.Count != classifier.MarshalMethods.Count) { + throw new InvalidOperationException ( + $"Internal error: classifier for template architecture '{templateState.TargetArch}' contains {templateClassifier.MarshalMethods.Count} marshal methods, but the one for architecture '{state.TargetArch}' has {classifier.MarshalMethods.Count}" + ); + } + + var matchedTemplateMethods = new HashSet (); + var mismatchedMethods = new List (); + bool foundMismatches = false; + + foreach (var kvp in classifier.MarshalMethods) { + string key = kvp.Key; + IList methods = kvp.Value; + + logger.LogDebugMessage ($"Comparing marshal method '{key}' in architecture '{templateState.TargetArch}', with {methods.Count} overloads, against architecture '{state.TargetArch}'"); + + if (!templateClassifier.MarshalMethods.TryGetValue (key, out IList templateMethods)) { + logger.LogDebugMessage ($"Architecture '{state.TargetArch}' has marshal method '{key}' which does not exist in architecture '{templateState.TargetArch}'"); + foundMismatches = true; + continue; + } + + if (methods.Count != templateMethods.Count) { + logger.LogDebugMessage ($"Architecture '{state.TargetArch}' has an incorrect number of marshal method '{key}' overloads. Expected {templateMethods.Count}, but found {methods.Count}"); + continue; + } + + foreach (MarshalMethodEntry templateMethod in templateMethods) { + MarshalMethodEntry? match = null; + + foreach (MarshalMethodEntry method in methods) { + if (CheckWhetherMethodsMatch (logger, templateMethod, templateState.TargetArch, method, state.TargetArch)) { + match = method; + break; + } + } + + if (match == null) { + foundMismatches = true; + } + } + } + + if (!foundMismatches) { + return; + } + + logger.LogError ($"Architecture '{state.TargetArch}' doesn't match all marshal methods in architecture '{templateState.TargetArch}'. Please see detailed MSBuild logs for more information."); + } + + static bool CheckWhetherMethodsMatch (TaskLoggingHelper logger, MarshalMethodEntry templateMethod, AndroidTargetArch templateArch, MarshalMethodEntry method, AndroidTargetArch arch) + { + bool success = true; + string methodName = templateMethod.NativeCallback.FullName; + + if (!CheckWhetherTypesMatch (templateMethod.DeclaringType, method.DeclaringType)) { + logger.LogDebugMessage ($"Marshal method '{methodName}' for architecture '{arch}' should be declared in type '{templateMethod.DeclaringType.FullName}', but instead was declared in '{method.DeclaringType.FullName}'"); + success = false; + } + + bool skipJniCheck = false; + if (!CheckWhetherMembersMatch (logger, methodName, "native callback", templateMethod.NativeCallback, templateArch, method.NativeCallback, arch)) { + success = false; + + // This takes care of overloads for the same methods, and avoids false negatives below + skipJniCheck = true; + } + + if (!skipJniCheck) { + if (String.Compare (templateMethod.JniMethodName, method.JniMethodName, StringComparison.Ordinal) != 0) { + logger.LogDebugMessage ($"Marshal method '{methodName}' for architecture '{arch}' has a different JNI method name than architecture '{templateArch}':"); + logger.LogDebugMessage ($" Expected: '{templateMethod.JniMethodName}', found: '{method.JniMethodName}'"); + success = false; + } + + if (String.Compare (templateMethod.JniMethodSignature, method.JniMethodSignature, StringComparison.Ordinal) != 0) { + logger.LogDebugMessage ($"Marshal method '{methodName}' for architecture '{arch}' has a different JNI method signature than architecture '{templateArch}':"); + logger.LogDebugMessage ($" Expected: '{templateMethod.JniMethodSignature}', found: '{method.JniMethodSignature}'"); + success = false; + } + + if (String.Compare (templateMethod.JniTypeName, method.JniTypeName, StringComparison.Ordinal) != 0) { + logger.LogDebugMessage ($"Marshal method '{methodName}' for architecture '{arch}' has a different JNI type name than architecture '{templateArch}':"); + logger.LogDebugMessage ($" Expected: '{templateMethod.JniTypeName}', found: '{method.JniTypeName}'"); + success = false; + } + } + + if (templateMethod.IsSpecial) { + // Other method definitions will be `null`, so we can skip them + if (method.IsSpecial) { + return success; + } + + logger.LogDebugMessage ($"Marshal method '{templateMethod.NativeCallback.FullName}' is marked as special in architecture '{templateArch}', but not in architecture '{arch}'"); + return false; + } + + if (!CheckWhetherMembersMatch (logger, methodName, "connector", templateMethod.Connector, templateArch, method.Connector, arch)) { + success = false; + } + + if (!CheckWhetherMembersMatch (logger, methodName, "implemented", templateMethod.ImplementedMethod, templateArch, method.ImplementedMethod, arch)) { + success = false; + } + + if (!CheckWhetherMembersMatch (logger, methodName, "registered", templateMethod.RegisteredMethod, templateArch, method.RegisteredMethod, arch)) { + success = false; + } + + if (!CheckWhetherMembersMatch (logger, methodName, "callback backing field", templateMethod.CallbackField, templateArch, method.CallbackField, arch)) { + success = false; + } + + return success; + } + + static bool CheckWhetherMembersMatch (TaskLoggingHelper logger, string marshalMethodName, string description, MemberReference? templateMethod, AndroidTargetArch templateArch, MemberReference? method, AndroidTargetArch arch) + { + if (templateMethod == null) { + if (method == null) { + return true; + } + + logger.LogDebugMessage ($"Marshal method '{marshalMethodName}' component '{description}' is null in architecture '{templateArch}', but not null in architecture '{arch}'"); + return false; + } + + return String.Compare (templateMethod.FullName, method.FullName, StringComparison.Ordinal) == 0; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs index 292e7229d64..7965c8c13f7 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs @@ -255,7 +255,7 @@ void ReorderActivityAliases (TaskLoggingHelper log, XElement app) } } - public IList Merge (TaskLoggingHelper log, TypeDefinitionCache cache, List subclasses, string applicationClass, bool embed, string bundledWearApplicationName, IEnumerable mergedManifestDocuments) + public IList Merge (TaskLoggingHelper log, TypeDefinitionCache cache, List subclasses, string applicationClass, bool embed, string bundledWearApplicationName, IEnumerable mergedManifestDocuments) { var manifest = doc.Root; @@ -330,8 +330,7 @@ public IList Merge (TaskLoggingHelper log, TypeDefinitionCache cache, Li throw new InvalidOperationException (string.Format ("The targetSdkVersion ({0}) is not a valid API level", targetSdkVersion)); int targetSdkVersionValue = tryTargetSdkVersion.Value; - foreach (JavaType jt in subclasses) { - TypeDefinition t = jt.Type; + foreach (TypeDefinition t in subclasses) { if (t.IsAbstract) continue; @@ -568,7 +567,7 @@ Func GetGenerator (T return null; } - XElement CreateApplicationElement (XElement manifest, string applicationClass, List subclasses, TypeDefinitionCache cache) + XElement CreateApplicationElement (XElement manifest, string applicationClass, List subclasses, TypeDefinitionCache cache) { var application = manifest.Descendants ("application").FirstOrDefault (); @@ -592,8 +591,7 @@ XElement CreateApplicationElement (XElement manifest, string applicationClass, L List typeAttr = new List (); List typeUsesLibraryAttr = new List (); List typeUsesConfigurationAttr = new List (); - foreach (JavaType jt in subclasses) { - TypeDefinition t = jt.Type; + foreach (TypeDefinition t in subclasses) { ApplicationAttribute aa = ApplicationAttribute.FromCustomAttributeProvider (t, cache); if (aa == null) continue; @@ -925,7 +923,7 @@ void AddSupportsGLTextures (XElement application, TypeDefinitionCache cache) } } - void AddInstrumentations (XElement manifest, IList subclasses, int targetSdkVersion, TypeDefinitionCache cache) + void AddInstrumentations (XElement manifest, IList subclasses, int targetSdkVersion, TypeDefinitionCache cache) { var assemblyAttrs = Assemblies.SelectMany (path => InstrumentationAttribute.FromCustomAttributeProvider (Resolver.GetAssembly (path), cache)); @@ -938,8 +936,7 @@ void AddInstrumentations (XElement manifest, IList subclasses, int tar manifest.Add (ia.ToElement (PackageName, cache)); } - foreach (JavaType jt in subclasses) { - TypeDefinition type = jt.Type; + foreach (TypeDefinition type in subclasses) { if (type.IsSubclassOf ("Android.App.Instrumentation", cache)) { var xe = InstrumentationFromTypeDefinition (type, JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'), cache); if (xe != null) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs index 96063bf2126..9f1034bd6b8 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs @@ -2,11 +2,11 @@ using System.Collections.Generic; using System.IO; -using Java.Interop.Tools.Cecil; using Microsoft.Android.Build.Tasks; using Microsoft.Build.Utilities; using Mono.Cecil; using Mono.Cecil.Cil; +using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks { @@ -21,29 +21,25 @@ sealed class AssemblyImports public MethodReference WaitForBridgeProcessingMethod; } - IDictionary> methods; - ICollection uniqueAssemblies; - IDictionary assemblyPaths; - TaskLoggingHelper log; + readonly TaskLoggingHelper log; + readonly MarshalMethodsClassifier classifier; + readonly XAAssemblyResolver resolver; + readonly AndroidTargetArch targetArch; - public MarshalMethodsAssemblyRewriter (IDictionary> methods, ICollection uniqueAssemblies, IDictionary assemblyPaths, TaskLoggingHelper log) + public MarshalMethodsAssemblyRewriter (TaskLoggingHelper log, AndroidTargetArch targetArch, MarshalMethodsClassifier classifier, XAAssemblyResolver resolver) { - this.assemblyPaths = assemblyPaths; - this.methods = methods ?? throw new ArgumentNullException (nameof (methods)); - this.uniqueAssemblies = uniqueAssemblies ?? throw new ArgumentNullException (nameof (uniqueAssemblies)); this.log = log ?? throw new ArgumentNullException (nameof (log)); + this.targetArch = targetArch; + this.classifier = classifier ?? throw new ArgumentNullException (nameof (classifier));; + this.resolver = resolver ?? throw new ArgumentNullException (nameof (resolver));; } // TODO: do away with broken exception transitions, there's no point in supporting them - public void Rewrite (XAAssemblyResolver resolver, bool brokenExceptionTransitions) + public void Rewrite (bool brokenExceptionTransitions) { - if (resolver == null) { - throw new ArgumentNullException (nameof (resolver)); - } - AssemblyDefinition? monoAndroidRuntime = resolver.Resolve ("Mono.Android.Runtime"); if (monoAndroidRuntime == null) { - throw new InvalidOperationException ($"Internal error: unable to load the Mono.Android.Runtime assembly"); + throw new InvalidOperationException ($"[{targetArch}] Internal error: unable to load the Mono.Android.Runtime assembly"); } TypeDefinition runtime = FindType (monoAndroidRuntime, "Android.Runtime.AndroidRuntimeInternal", required: true)!; @@ -59,8 +55,9 @@ public void Rewrite (XAAssemblyResolver resolver, bool brokenExceptionTransition TypeDefinition systemException = FindType (corlib, "System.Exception", required: true); MethodDefinition unmanagedCallersOnlyAttributeCtor = GetUnmanagedCallersOnlyAttributeConstructor (resolver); + var assemblyImports = new Dictionary (); - foreach (AssemblyDefinition asm in uniqueAssemblies) { + foreach (AssemblyDefinition asm in classifier.Assemblies) { var imports = new AssemblyImports { MonoUnhandledExceptionMethod = asm.MainModule.ImportReference (monoUnhandledExceptionMethod), SystemException = asm.MainModule.ImportReference (systemException), @@ -72,10 +69,10 @@ public void Rewrite (XAAssemblyResolver resolver, bool brokenExceptionTransition assemblyImports.Add (asm, imports); } - log.LogDebugMessage ("Rewriting assemblies for marshal methods support"); + log.LogDebugMessage ($"[{targetArch}] Rewriting assemblies for marshal methods support"); var processedMethods = new Dictionary (StringComparer.Ordinal); - foreach (IList methodList in methods.Values) { + foreach (IList methodList in classifier.MarshalMethods.Values) { foreach (MarshalMethodEntry method in methodList) { string fullNativeCallbackName = method.NativeCallback.FullName; if (processedMethods.TryGetValue (fullNativeCallbackName, out MethodDefinition nativeCallbackWrapper)) { @@ -86,19 +83,19 @@ public void Rewrite (XAAssemblyResolver resolver, bool brokenExceptionTransition method.NativeCallbackWrapper = GenerateWrapper (method, assemblyImports, brokenExceptionTransitions); if (method.Connector != null) { if (method.Connector.IsStatic && method.Connector.IsPrivate) { - log.LogDebugMessage ($"Removing connector method {method.Connector.FullName}"); + log.LogDebugMessage ($"[{targetArch}] Removing connector method {method.Connector.FullName}"); method.Connector.DeclaringType?.Methods?.Remove (method.Connector); } else { - log.LogWarning ($"NOT removing connector method {method.Connector.FullName} because it's either not static or not private"); + log.LogWarning ($"[{targetArch}] NOT removing connector method {method.Connector.FullName} because it's either not static or not private"); } } if (method.CallbackField != null) { if (method.CallbackField.IsStatic && method.CallbackField.IsPrivate) { - log.LogDebugMessage ($"Removing callback delegate backing field {method.CallbackField.FullName}"); + log.LogDebugMessage ($"[{targetArch}] Removing callback delegate backing field {method.CallbackField.FullName}"); method.CallbackField.DeclaringType?.Fields?.Remove (method.CallbackField); } else { - log.LogWarning ($"NOT removing callback field {method.CallbackField.FullName} because it's either not static or not private"); + log.LogWarning ($"[{targetArch}] NOT removing callback field {method.CallbackField.FullName} because it's either not static or not private"); } } @@ -106,8 +103,12 @@ public void Rewrite (XAAssemblyResolver resolver, bool brokenExceptionTransition } } - foreach (AssemblyDefinition asm in uniqueAssemblies) { - string path = GetAssemblyPath (asm); + foreach (AssemblyDefinition asm in classifier.Assemblies) { + string? path = asm.MainModule.FileName; + if (String.IsNullOrEmpty (path)) { + throw new InvalidOperationException ($"[{targetArch}] Internal error: assembly '{asm}' does not specify path to its file"); + } + string pathPdb = Path.ChangeExtension (path, ".pdb"); bool havePdb = File.Exists (pathPdb); @@ -118,7 +119,7 @@ public void Rewrite (XAAssemblyResolver resolver, bool brokenExceptionTransition string directory = Path.Combine (Path.GetDirectoryName (path), "new"); Directory.CreateDirectory (directory); string output = Path.Combine (directory, Path.GetFileName (path)); - log.LogDebugMessage ($"Writing new version of '{path}' assembly: {output}"); + log.LogDebugMessage ($"[{targetArch}] Writing new version of '{path}' assembly: {output}"); // TODO: this should be used eventually, but it requires that all the types are reloaded from the assemblies before typemaps are generated // since Cecil doesn't update the MVID in the already loaded types @@ -139,7 +140,7 @@ public void Rewrite (XAAssemblyResolver resolver, bool brokenExceptionTransition void CopyFile (string source, string target) { - log.LogDebugMessage ($"Copying rewritten assembly: {source} -> {target}"); + log.LogDebugMessage ($"[{targetArch}] Copying rewritten assembly: {source} -> {target}"); string targetBackup = $"{target}.bak"; if (File.Exists (target)) { @@ -154,8 +155,8 @@ void CopyFile (string source, string target) File.Delete (targetBackup); } catch (Exception ex) { // On Windows the deletion may fail, depending on lock state of the original `target` file before the move. - log.LogDebugMessage ($"While trying to delete '{targetBackup}', exception was thrown: {ex}"); - log.LogDebugMessage ($"Failed to delete backup file '{targetBackup}', ignoring."); + log.LogDebugMessage ($"[{targetArch}] While trying to delete '{targetBackup}', exception was thrown: {ex}"); + log.LogDebugMessage ($"[{targetArch}] Failed to delete backup file '{targetBackup}', ignoring."); } } } @@ -167,11 +168,11 @@ void RemoveFile (string? path) } try { - log.LogDebugMessage ($"Deleting: {path}"); + log.LogDebugMessage ($"[{targetArch}] Deleting: {path}"); File.Delete (path); } catch (Exception ex) { - log.LogWarning ($"Unable to delete source file '{path}'"); - log.LogDebugMessage (ex.ToString ()); + log.LogWarning ($"[{targetArch}] Unable to delete source file '{path}'"); + log.LogDebugMessage ($"[{targetArch}] {ex.ToString ()}"); } } } @@ -323,7 +324,7 @@ bool IsBooleanConversion (TypeReference sourceType, TypeReference targetType) { if (String.Compare ("System.Boolean", sourceType.FullName, StringComparison.Ordinal) == 0) { if (String.Compare ("System.Byte", targetType.FullName, StringComparison.Ordinal) != 0) { - throw new InvalidOperationException ($"Unexpected conversion from '{sourceType.FullName}' to '{targetType.FullName}'"); + throw new InvalidOperationException ($"[{targetArch}] Unexpected conversion from '{sourceType.FullName}' to '{targetType.FullName}'"); } return true; @@ -334,7 +335,7 @@ bool IsBooleanConversion (TypeReference sourceType, TypeReference targetType) void ThrowUnsupportedType (TypeReference type) { - throw new InvalidOperationException ($"Unsupported non-blittable type '{type.FullName}'"); + throw new InvalidOperationException ($"[{targetArch}] Unsupported non-blittable type '{type.FullName}'"); } } @@ -384,7 +385,7 @@ void AddSetDefaultValueInstructions (MethodBody body, TypeReference type, Variab return; } - throw new InvalidOperationException ($"Unsupported type: '{type.FullName}'"); + throw new InvalidOperationException ($"[{targetArch}] Unsupported type: '{type.FullName}'"); } @@ -436,31 +437,20 @@ TypeReference MapToBlittableTypeIfNecessary (TypeReference type, out bool typeMa return ReturnValid (typeof(byte)); } - throw new NotSupportedException ($"Cannot map unsupported blittable type '{type.FullName}'"); + throw new NotSupportedException ($"[{targetArch}] Cannot map unsupported blittable type '{type.FullName}'"); TypeReference ReturnValid (Type typeToLookUp) { TypeReference? mappedType = type.Module.Assembly.MainModule.ImportReference (typeToLookUp); if (mappedType == null) { - throw new InvalidOperationException ($"Unable to obtain reference to type '{typeToLookUp.FullName}'"); + throw new InvalidOperationException ($"[{targetArch}] Unable to obtain reference to type '{typeToLookUp.FullName}'"); } return mappedType; } } - string GetAssemblyPath (AssemblyDefinition asm) - { - string filePath = asm.MainModule.FileName; - if (!String.IsNullOrEmpty (filePath)) { - return filePath; - } - - // No checking on purpose - the assembly **must** be there if its MainModule.FileName property returns a null or empty string - return assemblyPaths[asm]; - } - - MethodDefinition GetUnmanagedCallersOnlyAttributeConstructor (XAAssemblyResolver resolver) + MethodDefinition GetUnmanagedCallersOnlyAttributeConstructor (IAssemblyResolver resolver) { AssemblyDefinition asm = resolver.Resolve ("System.Runtime.InteropServices"); TypeDefinition unmanagedCallersOnlyAttribute = null; @@ -480,7 +470,7 @@ MethodDefinition GetUnmanagedCallersOnlyAttributeConstructor (XAAssemblyResolver } if (unmanagedCallersOnlyAttribute == null) { - throw new InvalidOperationException ("Unable to find the System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute type"); + throw new InvalidOperationException ("[{targetArch}] Unable to find the System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute type"); } foreach (MethodDefinition md in unmanagedCallersOnlyAttribute.Methods) { @@ -491,7 +481,7 @@ MethodDefinition GetUnmanagedCallersOnlyAttributeConstructor (XAAssemblyResolver return md; } - throw new InvalidOperationException ("Unable to find the System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute type constructor"); + throw new InvalidOperationException ("[{targetArch}] Unable to find the System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute type constructor"); } CustomAttribute CreateImportedUnmanagedCallersOnlyAttribute (AssemblyDefinition targetAssembly, MethodDefinition unmanagedCallersOnlyAtributeCtor) @@ -501,17 +491,17 @@ CustomAttribute CreateImportedUnmanagedCallersOnlyAttribute (AssemblyDefinition MethodDefinition? FindMethod (TypeDefinition type, string methodName, bool required) { - log.LogDebugMessage ($"Looking for method '{methodName}' in type {type}"); + log.LogDebugMessage ($"[{targetArch}] Looking for method '{methodName}' in type {type}"); foreach (MethodDefinition method in type.Methods) { - log.LogDebugMessage ($" method: {method.Name}"); + log.LogDebugMessage ($"[{targetArch}] method: {method.Name}"); if (String.Compare (methodName, method.Name, StringComparison.Ordinal) == 0) { - log.LogDebugMessage (" match!"); + log.LogDebugMessage ($"[{targetArch}] match!"); return method; } } if (required) { - throw new InvalidOperationException ($"Internal error: required method '{methodName}' in type {type} not found"); + throw new InvalidOperationException ($"[{targetArch}] Internal error: required method '{methodName}' in type {type} not found"); } return null; @@ -519,20 +509,30 @@ CustomAttribute CreateImportedUnmanagedCallersOnlyAttribute (AssemblyDefinition TypeDefinition? FindType (AssemblyDefinition asm, string typeName, bool required) { - log.LogDebugMessage ($"Looking for type '{typeName}' in assembly '{asm}'"); + log.LogDebugMessage ($"[{targetArch}] Looking for type '{typeName}' in assembly '{asm}' ({GetAssemblyPathInfo (asm)})"); foreach (TypeDefinition t in asm.MainModule.Types) { - log.LogDebugMessage ($" checking {t.FullName}"); + log.LogDebugMessage ($"[{targetArch}] checking {t.FullName}"); if (String.Compare (typeName, t.FullName, StringComparison.Ordinal) == 0) { - log.LogDebugMessage ($" match!"); + log.LogDebugMessage ($"[{targetArch}] match!"); return t; } } if (required) { - throw new InvalidOperationException ($"Internal error: required type '{typeName}' in assembly {asm} not found"); + throw new InvalidOperationException ($"[{targetArch}] Internal error: required type '{typeName}' in assembly {asm} not found"); } return null; } + + static string GetAssemblyPathInfo (AssemblyDefinition asm) + { + string? path = asm.MainModule.FileName; + if (String.IsNullOrEmpty (path)) { + return "no assembly path"; + } + + return path; + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs index 0792f992daa..1507084dfc0 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs @@ -9,6 +9,7 @@ using Microsoft.Android.Build.Tasks; using Microsoft.Build.Utilities; using Mono.Cecil; +using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks { @@ -80,6 +81,13 @@ string EnsureNonEmpty (string s, string argName) return s; } + + public string GetStoreMethodKey (TypeDefinitionCache tdCache) + { + MethodDefinition registeredMethod = RegisteredMethod; + string typeName = registeredMethod.DeclaringType.FullName.Replace ('/', '+'); + return $"{typeName}, {registeredMethod.DeclaringType.GetPartialAssemblyName (tdCache)}\t{registeredMethod.Name}"; + } } class MarshalMethodsClassifier : JavaCallableMethodClassifier @@ -226,20 +234,33 @@ public bool Matches (MethodDefinition method) } TypeDefinitionCache tdCache; - XAAssemblyResolver resolver; + IAssemblyResolver resolver; Dictionary> marshalMethods; HashSet assemblies; TaskLoggingHelper log; HashSet typesWithDynamicallyRegisteredMethods; ulong rejectedMethodCount = 0; ulong wrappedMethodCount = 0; + readonly AndroidTargetArch targetArch; public IDictionary> MarshalMethods => marshalMethods; public ICollection Assemblies => assemblies; public ulong RejectedMethodCount => rejectedMethodCount; public ulong WrappedMethodCount => wrappedMethodCount; + public TypeDefinitionCache TypeDefinitionCache => tdCache; + + public MarshalMethodsClassifier (AndroidTargetArch targetArch, TypeDefinitionCache tdCache, IAssemblyResolver res, TaskLoggingHelper log) + { + this.targetArch = targetArch; + this.log = log ?? throw new ArgumentNullException (nameof (log)); + this.tdCache = tdCache ?? throw new ArgumentNullException (nameof (tdCache)); + resolver = res ?? throw new ArgumentNullException (nameof (tdCache)); + marshalMethods = new Dictionary> (StringComparer.Ordinal); + assemblies = new HashSet (); + typesWithDynamicallyRegisteredMethods = new HashSet (); + } - public MarshalMethodsClassifier (TypeDefinitionCache tdCache, XAAssemblyResolver res, TaskLoggingHelper log) + public MarshalMethodsClassifier (TypeDefinitionCache tdCache, IAssemblyResolver res, TaskLoggingHelper log) { this.log = log ?? throw new ArgumentNullException (nameof (log)); this.tdCache = tdCache ?? throw new ArgumentNullException (nameof (tdCache)); @@ -404,6 +425,24 @@ public void AddSpecialCaseMethods () AddTypeManagerSpecialCaseMethods (); } + string GetAssemblyPathInfo (FieldDefinition? field) => GetAssemblyPathInfo (field?.DeclaringType); + string GetAssemblyPathInfo (MethodDefinition? method) => GetAssemblyPathInfo (method?.DeclaringType); + string GetAssemblyPathInfo (TypeDefinition? type) => GetAssemblyPathInfo (type?.Module?.Assembly); + + string GetAssemblyPathInfo (AssemblyDefinition? asmdef) + { + if (asmdef == null) { + return "[assembly definition missing]"; + } + + string? path = asmdef.MainModule.FileName; + if (String.IsNullOrEmpty (path)) { + path = "unknown"; + } + + return $"[Arch: {targetArch}; Assembly: {path}]"; + } + bool IsDynamicallyRegistered (TypeDefinition topType, MethodDefinition registeredMethod, MethodDefinition implementedMethod, CustomAttribute registerAttribute) { if (registerAttribute.ConstructorArguments.Count != 3) { @@ -417,7 +456,7 @@ bool IsDynamicallyRegistered (TypeDefinition topType, MethodDefinition registere return false; } - log.LogWarning ($"Method '{registeredMethod.FullName}' will be registered dynamically"); + log.LogWarning ($"Method '{registeredMethod.FullName}' will be registered dynamically {GetAssemblyPathInfo (registeredMethod)}"); rejectedMethodCount++; return true; } @@ -431,7 +470,7 @@ bool IsStandardHandler (TypeDefinition topType, ConnectorInfo connector, MethodD if (connectorName.Length < HandlerNameStart.Length + HandlerNameEnd.Length + 1 || !connectorName.StartsWith (HandlerNameStart, StringComparison.Ordinal) || !connectorName.EndsWith (HandlerNameEnd, StringComparison.Ordinal)) { - log.LogWarning ($"\tConnector name '{connectorName}' must start with '{HandlerNameStart}', end with '{HandlerNameEnd}' and have at least one character between the two parts."); + log.LogWarning ($"Connector name '{connectorName}' must start with '{HandlerNameStart}', end with '{HandlerNameEnd}' and have at least one character between the two parts."); return false; } @@ -446,19 +485,19 @@ bool IsStandardHandler (TypeDefinition topType, ConnectorInfo connector, MethodD MethodDefinition connectorMethod = FindMethod (connectorDeclaringType, connectorName); if (connectorMethod == null) { - log.LogWarning ($"\tConnector method '{connectorName}' not found in type '{connectorDeclaringType.FullName}'"); + log.LogWarning ($"Connector method '{connectorName}' not found in type '{connectorDeclaringType.FullName}' {GetAssemblyPathInfo (connectorDeclaringType)}"); return false; } if (String.Compare ("System.Delegate", connectorMethod.ReturnType.FullName, StringComparison.Ordinal) != 0) { - log.LogWarning ($"\tConnector '{connectorName}' in type '{connectorDeclaringType.FullName}' has invalid return type, expected 'System.Delegate', found '{connectorMethod.ReturnType.FullName}'"); + log.LogWarning ($"Connector '{connectorName}' in type '{connectorDeclaringType.FullName}' has invalid return type, expected 'System.Delegate', found '{connectorMethod.ReturnType.FullName}' {GetAssemblyPathInfo (connectorDeclaringType)}"); return false; } var ncbs = new NativeCallbackSignature (registeredMethod, log, tdCache); MethodDefinition nativeCallbackMethod = FindMethod (connectorDeclaringType, nativeCallbackName, ncbs); if (nativeCallbackMethod == null) { - log.LogWarning ($"\tUnable to find native callback method '{nativeCallbackName}' in type '{connectorDeclaringType.FullName}', matching the '{registeredMethod.FullName}' signature (jniName: '{jniName}')"); + log.LogWarning ($"Unable to find native callback method '{nativeCallbackName}' in type '{connectorDeclaringType.FullName}', matching the '{registeredMethod.FullName}' signature (jniName: '{jniName}') {GetAssemblyPathInfo (connectorDeclaringType)}"); return false; } @@ -471,7 +510,7 @@ bool IsStandardHandler (TypeDefinition topType, ConnectorInfo connector, MethodD FieldDefinition delegateField = FindField (nativeCallbackMethod.DeclaringType, delegateFieldName); if (delegateField != null) { if (String.Compare ("System.Delegate", delegateField.FieldType.FullName, StringComparison.Ordinal) != 0) { - log.LogWarning ($"\tdelegate field '{delegateFieldName}' in type '{nativeCallbackMethod.DeclaringType.FullName}' has invalid type, expected 'System.Delegate', found '{delegateField.FieldType.FullName}'"); + log.LogWarning ($"delegate field '{delegateFieldName}' in type '{nativeCallbackMethod.DeclaringType.FullName}' has invalid type, expected 'System.Delegate', found '{delegateField.FieldType.FullName}' {GetAssemblyPathInfo (delegateField)}"); return false; } } @@ -690,16 +729,9 @@ FieldDefinition FindField (TypeDefinition type, string fieldName, bool lookForIn return FindField (tdCache.Resolve (type.BaseType), fieldName, lookForInherited); } - public string GetStoreMethodKey (MarshalMethodEntry methodEntry) - { - MethodDefinition registeredMethod = methodEntry.RegisteredMethod; - string typeName = registeredMethod.DeclaringType.FullName.Replace ('/', '+'); - return $"{typeName}, {registeredMethod.DeclaringType.GetPartialAssemblyName (tdCache)}\t{registeredMethod.Name}"; - } - void StoreMethod (MarshalMethodEntry entry) { - string key = GetStoreMethodKey (entry); + string key = entry.GetStoreMethodKey (tdCache); // Several classes can override the same method, we need to generate the marshal method only once, at the same time // keeping track of overloads diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs index ee6fc0e7f28..46faa8d7479 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs @@ -8,6 +8,7 @@ using Microsoft.Build.Utilities; using Xamarin.Android.Tasks.LLVMIR; +using Xamarin.Android.Tools; using CecilMethodDefinition = global::Mono.Cecil.MethodDefinition; using CecilParameterDefinition = global::Mono.Cecil.ParameterDefinition; @@ -135,7 +136,7 @@ public override string GetComment (object data, string fieldName) var methodName = EnsureType (data); if (String.Compare ("id", fieldName, StringComparison.Ordinal) == 0) { - return $" id 0x{methodName.id:x}; name: {methodName.name}"; + return $" name: {methodName.name}"; } return String.Empty; @@ -151,7 +152,7 @@ sealed class MarshalMethodName [NativeAssembler (Ignore = true)] public ulong Id64; - [NativeAssembler (UsesDataProvider = true)] + [NativeAssembler (UsesDataProvider = true, NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal)] public ulong id; public string name; } @@ -225,9 +226,8 @@ public MarshalMethodAssemblyIndexValuePlaceholder (MarshalMethodInfo mmi, Assemb { 'L', typeof(_jobjectArray) }, }; - ICollection uniqueAssemblyNames; - int numberOfAssembliesInApk; - IDictionary> marshalMethods; + readonly ICollection uniqueAssemblyNames; + readonly int numberOfAssembliesInApk; StructureInfo marshalMethodsManagedClassStructureInfo; StructureInfo marshalMethodNameStructureInfo; @@ -235,16 +235,18 @@ public MarshalMethodAssemblyIndexValuePlaceholder (MarshalMethodInfo mmi, Assemb List methods; List> classes = new List> (); - LlvmIrCallMarker defaultCallMarker; - + readonly LlvmIrCallMarker defaultCallMarker; readonly bool generateEmptyCode; + readonly AndroidTargetArch targetArch; + readonly NativeCodeGenState? codeGenState; /// /// Constructor to be used ONLY when marshal methods are DISABLED /// - public MarshalMethodsNativeAssemblyGenerator (TaskLoggingHelper log, int numberOfAssembliesInApk, ICollection uniqueAssemblyNames) + public MarshalMethodsNativeAssemblyGenerator (TaskLoggingHelper log, AndroidTargetArch targetArch, int numberOfAssembliesInApk, ICollection uniqueAssemblyNames) : base (log) { + this.targetArch = targetArch; this.numberOfAssembliesInApk = numberOfAssembliesInApk; this.uniqueAssemblyNames = uniqueAssemblyNames ?? throw new ArgumentNullException (nameof (uniqueAssemblyNames)); generateEmptyCode = true; @@ -254,12 +256,12 @@ public MarshalMethodsNativeAssemblyGenerator (TaskLoggingHelper log, int numberO /// /// Constructor to be used ONLY when marshal methods are ENABLED /// - public MarshalMethodsNativeAssemblyGenerator (TaskLoggingHelper log, int numberOfAssembliesInApk, ICollection uniqueAssemblyNames, IDictionary> marshalMethods) + public MarshalMethodsNativeAssemblyGenerator (TaskLoggingHelper log, int numberOfAssembliesInApk, ICollection uniqueAssemblyNames, NativeCodeGenState codeGenState) : base (log) { this.numberOfAssembliesInApk = numberOfAssembliesInApk; this.uniqueAssemblyNames = uniqueAssemblyNames ?? throw new ArgumentNullException (nameof (uniqueAssemblyNames)); - this.marshalMethods = marshalMethods; + this.codeGenState = codeGenState ?? throw new ArgumentNullException (nameof (codeGenState)); generateEmptyCode = false; defaultCallMarker = LlvmIrCallMarker.Tail; @@ -267,12 +269,13 @@ public MarshalMethodsNativeAssemblyGenerator (TaskLoggingHelper log, int numberO void Init () { - if (generateEmptyCode || marshalMethods == null || marshalMethods.Count == 0) { + if (generateEmptyCode || codeGenState.Classifier == null || codeGenState.Classifier.MarshalMethods.Count == 0) { return; } var seenClasses = new Dictionary (StringComparer.Ordinal); var allMethods = new List (); + IDictionary> marshalMethods = codeGenState.Classifier.MarshalMethods; // It's possible that several otherwise different methods (from different classes, but with the same // names and similar signatures) will actually share the same **short** native symbol name. In this case we must @@ -305,6 +308,7 @@ void Init () foreach (IList entryList in marshalMethods.Values) { bool useFullNativeSignature = entryList.Count > 1; foreach (MarshalMethodEntry entry in entryList) { + Log.LogDebugMessage ($"MM: processing {entry.DeclaringType.FullName} {entry.NativeCallback.FullName}"); ProcessAndAddMethod (allMethods, entry, useFullNativeSignature, seenClasses, overloadedNativeSymbolNames); } } @@ -654,6 +658,7 @@ void AddMarshalMethods (LlvmIrModule module, AssemblyCacheState acs, LlvmIrVaria void AddMarshalMethod (LlvmIrModule module, MarshalMethodInfo method, ulong asmId, MarshalMethodsWriteState writeState) { + Log.LogDebugMessage ($"MM: generating code for {method.Method.DeclaringType.FullName} {method.Method.NativeCallback.FullName}"); CecilMethodDefinition nativeCallback = method.Method.NativeCallback; string backingFieldName = $"native_cb_{method.Method.JniMethodName}_{asmId}_{method.ClassCacheIndex}_{nativeCallback.MetadataToken.ToUInt32():x}"; @@ -848,12 +853,11 @@ LlvmIrFunctionAttributeSet MakeXamarinAppInitAttributeSet (LlvmIrModule module) new NosyncFunctionAttribute (), new NounwindFunctionAttribute (), new WillreturnFunctionAttribute (), - // TODO: LLVM 16+ feature, enable when we switch to this version - // new MemoryFunctionAttribute { - // Default = MemoryAttributeAccessKind.Write, - // Argmem = MemoryAttributeAccessKind.None, - // InaccessibleMem = MemoryAttributeAccessKind.None, - // }, + new MemoryFunctionAttribute { + Default = MemoryAttributeAccessKind.Write, + Argmem = MemoryAttributeAccessKind.None, + InaccessibleMem = MemoryAttributeAccessKind.None, + }, new UwtableFunctionAttribute (), new MinLegalVectorWidthFunctionAttribute (0), new NoTrappingMathFunctionAttribute (true), @@ -981,10 +985,24 @@ void AddAssemblyImageCache (LlvmIrModule module, out AssemblyCacheState acs) foreach (string name in uniqueAssemblyNames) { // We must make sure we keep the possible culture prefix, which will be treated as "directory" path here - string clippedName = Path.Combine (Path.GetDirectoryName (name) ?? String.Empty, Path.GetFileNameWithoutExtension (name)); + string cultureName = Path.GetDirectoryName (name) ?? String.Empty; + string clippedName = Path.Combine (cultureName, Path.GetFileNameWithoutExtension (name)); + string inArchiveName; + + if (cultureName.Length == 0) { + // Regular assemblies get the 'lib_' prefix + inArchiveName = $"{MonoAndroidHelper.MANGLED_ASSEMBLY_REGULAR_ASSEMBLY_MARKER}{name}{MonoAndroidHelper.MANGLED_ASSEMBLY_NAME_EXT}"; + } else { + // Satellite assemblies get the 'lib-{CULTURE}-' prefix + inArchiveName = $"{MonoAndroidHelper.MANGLED_ASSEMBLY_SATELLITE_ASSEMBLY_MARKER}{cultureName}-{Path.GetFileName (name)}{MonoAndroidHelper.MANGLED_ASSEMBLY_NAME_EXT}"; + } + ulong hashFull32 = MonoAndroidHelper.GetXxHash (name, is64Bit: false); + ulong hashInArchive32 = MonoAndroidHelper.GetXxHash (inArchiveName, is64Bit: false); ulong hashClipped32 = MonoAndroidHelper.GetXxHash (clippedName, is64Bit: false); + ulong hashFull64 = MonoAndroidHelper.GetXxHash (name, is64Bit: true); + ulong hashInArchive64 = MonoAndroidHelper.GetXxHash (inArchiveName, is64Bit: true); ulong hashClipped64 = MonoAndroidHelper.GetXxHash (clippedName, is64Bit: true); // @@ -992,8 +1010,11 @@ void AddAssemblyImageCache (LlvmIrModule module, out AssemblyCacheState acs) // `number_of_assembly_name_forms_in_image_cache` constant to the number of forms. // acs.Hashes32.Add ((uint)Convert.ChangeType (hashFull32, typeof(uint)), (name, index)); + acs.Hashes32.Add ((uint)Convert.ChangeType (hashInArchive32, typeof(uint)), (inArchiveName, index)); acs.Hashes32.Add ((uint)Convert.ChangeType (hashClipped32, typeof(uint)), (clippedName, index)); + acs.Hashes64.Add (hashFull64, (name, index)); + acs.Hashes64.Add (hashInArchive64, (inArchiveName, index)); acs.Hashes64.Add (hashClipped64, (clippedName, index)); index++; @@ -1067,7 +1088,7 @@ void UpdateAssemblyImageCacheHashes (LlvmIrVariable variable, LlvmIrModuleTarget i = acs.Hashes32[v32].index; } - return $" {index}: {name} => 0x{value:x} => {i}"; + return $" {index}: {name} => {i}"; } void UpdateAssemblyImageCacheIndices (LlvmIrVariable variable, LlvmIrModuleTarget target, object? callerState) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsState.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsState.cs index 66bf0fdba2e..03668cc2240 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsState.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsState.cs @@ -9,6 +9,8 @@ sealed class MarshalMethodsState public MarshalMethodsState (IDictionary> marshalMethods) { + MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified ethods in MarshalMethodsState ctor", marshalMethods); + MarshalMethods = marshalMethods ?? throw new ArgumentNullException (nameof (marshalMethods)); } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Basic.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Basic.cs index 3e8fe9a2d8c..09fb51317c8 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Basic.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Basic.cs @@ -14,16 +14,16 @@ public static class AndroidAbi { public const string Arm32 = "armeabi-v7a"; public const string Arm64 = "arm64-v8a"; - public const string X86 = "x86"; - public const string X64 = "x86_64"; + public const string X86 = "x86"; + public const string X64 = "x86_64"; } public static class RuntimeIdentifier { public const string Arm32 = "android-arm"; public const string Arm64 = "android-arm64"; - public const string X86 = "android-x86"; - public const string X64 = "android-x64"; + public const string X86 = "android-x86"; + public const string X64 = "android-x64"; } public static readonly HashSet SupportedTargetArchitectures = new HashSet { @@ -36,10 +36,10 @@ public static class RuntimeIdentifier static readonly char[] ZipPathTrimmedChars = {'/', '\\'}; static readonly Dictionary ClangAbiMap = new Dictionary (StringComparer.OrdinalIgnoreCase) { - {"arm64-v8a", "aarch64"}, + {"arm64-v8a", "aarch64"}, {"armeabi-v7a", "arm"}, - {"x86", "i686"}, - {"x86_64", "x86_64"} + {"x86", "i686"}, + {"x86_64", "x86_64"} }; static readonly Dictionary AbiToArchMap = new Dictionary (StringComparer.OrdinalIgnoreCase) { @@ -87,7 +87,7 @@ public static class RuntimeIdentifier public static AndroidTargetArch AbiToTargetArch (string abi) { if (!AbiToArchMap.TryGetValue (abi, out AndroidTargetArch arch)) { - return AndroidTargetArch.None; + throw new NotSupportedException ($"Internal error: unsupported ABI '{abi}'"); }; return arch; @@ -138,6 +138,8 @@ public static string ArchToAbi (AndroidTargetArch arch) return abi; } + public static bool IsValidAbi (string abi) => AbiToRidMap.ContainsKey (abi); + public static string? CultureInvariantToString (object? obj) { if (obj == null) { @@ -165,7 +167,7 @@ public static string MakeZipArchivePath (string part1, ICollection? path var parts = new List (); if (!String.IsNullOrEmpty (part1)) { parts.Add (part1.TrimEnd (ZipPathTrimmedChars)); - }; + }; if (pathParts != null && pathParts.Count > 0) { foreach (string p in pathParts) { @@ -183,7 +185,36 @@ public static string MakeZipArchivePath (string part1, ICollection? path return String.Join ("/", parts); } - public static bool IsValidAbi (string abi) => AbiToRidMap.ContainsKey (abi); + // These 3 MUST be the same as the like-named constants in src/monodroid/jni/shared-constants.hh + public const string MANGLED_ASSEMBLY_NAME_EXT = ".so"; + public const string MANGLED_ASSEMBLY_REGULAR_ASSEMBLY_MARKER = "lib_"; + public const string MANGLED_ASSEMBLY_SATELLITE_ASSEMBLY_MARKER = "lib-"; + + /// + /// Mangles APK/AAB entry name for assembly and their associated pdb and config entries in the + /// way expected by our native runtime. Must **NOT** be used to mangle names when assembly stores + /// are used. Must **NOT** be used for entries other than assemblies and their associated files. + /// + public static string MakeDiscreteAssembliesEntryName (string name, string? culture = null) + { + if (!String.IsNullOrEmpty (culture)) { + return $"{MANGLED_ASSEMBLY_SATELLITE_ASSEMBLY_MARKER}{culture}-{name}{MANGLED_ASSEMBLY_NAME_EXT}"; + } + + return $"{MANGLED_ASSEMBLY_REGULAR_ASSEMBLY_MARKER}{name}{MANGLED_ASSEMBLY_NAME_EXT}"; + } + + /// + /// Returns size of the extension + length of the prefix for mangled assembly names. This is + /// used to pre-allocate space for assembly names in `libxamarin-app.so` + /// + /// + public static ulong GetMangledAssemblyNameSizeOverhead () + { + // Satellite marker is one character more, for the `-` closing the culture part + return (ulong)MANGLED_ASSEMBLY_NAME_EXT.Length + + (ulong)Math.Max (MANGLED_ASSEMBLY_SATELLITE_ASSEMBLY_MARKER.Length + 1, MANGLED_ASSEMBLY_REGULAR_ASSEMBLY_MARKER.Length); + } public static byte[] Utf8StringToBytes (string str) => Encoding.UTF8.GetBytes (str); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index 9c2794a367a..fb598eadf2a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -542,9 +542,14 @@ public static int ConvertSupportedOSPlatformVersionToApiLevel (string version) } #if MSBUILD - public static string? GetAssemblyAbi (ITaskItem asmItem) + public static string GetAssemblyAbi (ITaskItem asmItem) { - return asmItem.GetMetadata ("Abi"); + string? abi = asmItem.GetMetadata ("Abi"); + if (String.IsNullOrEmpty (abi)) { + throw new InvalidOperationException ($"Internal error: assembly '{asmItem}' lacks ABI metadata"); + } + + return abi; } public static AndroidTargetArch GetTargetArch (ITaskItem asmItem) => AbiToTargetArch (GetAssemblyAbi (asmItem)); @@ -579,5 +584,108 @@ public static string GetNativeLibsRootDirectoryPath (string androidBinUtilsDirec string relPath = GetToolsRootDirectoryRelativePath (androidBinUtilsDirectory); return Path.GetFullPath (Path.Combine (androidBinUtilsDirectory, relPath, "lib")); } + + /// + /// Process a collection of assembly `ITaskItem` objects, splitting it on the assembly architecture () while, at the same time, ignoring + /// all assemblies which are **not** in the collection. If necessary, the selection can be further controlled by passing a qualifier + /// function in which returns `true` if the assembly passed to it should be **skipped**. + /// + /// This method is necessary because sometimes our tasks will be given assemblies for more architectures than indicated as supported in their `SupportedAbis` properties. + /// One such example is the `AotTests.BuildAMassiveApp` test, which passes around a set of assemblies for all the supported architectures, but it supports only two ABIs + /// via the `SupportedAbis` property. + /// + public static Dictionary> GetPerArchAssemblies (IEnumerable input, ICollection supportedAbis, bool validate, Func? shouldSkip = null) + { + var supportedTargetArches = new HashSet (); + foreach (string abi in supportedAbis) { + supportedTargetArches.Add (AbiToTargetArch (abi)); + } + + return GetPerArchAssemblies ( + input, + supportedTargetArches, + validate, + shouldSkip + ); + } + + static Dictionary> GetPerArchAssemblies (IEnumerable input, HashSet supportedTargetArches, bool validate, Func? shouldSkip = null) + { + bool filterByTargetArches = supportedTargetArches.Count > 0; + var assembliesPerArch = new Dictionary> (); + foreach (ITaskItem assembly in input) { + if (shouldSkip != null && shouldSkip (assembly)) { + continue; + } + + AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (assembly); + if (filterByTargetArches && !supportedTargetArches.Contains (arch)) { + continue; + } + + if (!assembliesPerArch.TryGetValue (arch, out Dictionary assemblies)) { + assemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); + assembliesPerArch.Add (arch, assemblies); + } + + string name = Path.GetFileNameWithoutExtension (assembly.ItemSpec); + string? culture = assembly.GetMetadata ("Culture"); + if (!String.IsNullOrEmpty (culture)) { + name = $"{culture}/{name}"; + } + assemblies.Add (name, assembly); + } + + // It's possible some assembly collections will be empty (e.g. `ResolvedUserAssemblies` as passed to the `GenerateJavaStubs` task), which + // isn't a problem and such empty collections should not be validated, as it will end in the "should never happen" exception below being + // thrown as a false negative. + if (assembliesPerArch.Count == 0 || !validate) { + return assembliesPerArch; + } + + Dictionary? firstArchAssemblies = null; + AndroidTargetArch firstArch = AndroidTargetArch.None; + foreach (var kvp in assembliesPerArch) { + if (firstArchAssemblies == null) { + firstArchAssemblies = kvp.Value; + firstArch = kvp.Key; + continue; + } + + EnsureDictionariesHaveTheSameEntries (firstArchAssemblies, kvp.Value, kvp.Key); + } + + // Should "never" happen... + if (firstArch == AndroidTargetArch.None) { + throw new InvalidOperationException ("Internal error: no per-architecture assemblies found?"); + } + + return assembliesPerArch; + + void EnsureDictionariesHaveTheSameEntries (Dictionary template, Dictionary dict, AndroidTargetArch arch) + { + if (dict.Count != template.Count) { + throw new InvalidOperationException ($"Internal error: architecture '{arch}' should have {template.Count} assemblies, however it has {dict.Count}"); + } + + foreach (var kvp in template) { + if (!dict.ContainsKey (kvp.Key)) { + throw new InvalidOperationException ($"Internal error: architecture '{arch}' does not have assembly '{kvp.Key}'"); + } + } + } + } + + internal static void DumpMarshalMethodsToConsole (string heading, IDictionary> marshalMethods) + { + Console.WriteLine (); + Console.WriteLine ($"{heading}:"); + foreach (var kvp in marshalMethods) { + Console.WriteLine ($" {kvp.Key}"); + foreach (var method in kvp.Value) { + Console.WriteLine ($" {method.DeclaringType.FullName} {method.NativeCallback.FullName}"); + } + } + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs new file mode 100644 index 00000000000..96d95391a49 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs @@ -0,0 +1,47 @@ +using System.Collections.Generic; + +using Java.Interop.Tools.Cecil; +using Mono.Cecil; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +/// +/// Holds state for typemap and marshal methods generators. A single instance of this +/// class is created for each enabled target architecture. +/// +class NativeCodeGenState +{ + public static NativeCodeGenState? Template { get; set; } + + /// + /// Target architecture for which this instance was created. + /// + public AndroidTargetArch TargetArch { get; } + + /// + /// Classifier used when scanning for Java types in the target architecture's + /// assemblies. Will be **null** if marshal methods are disabled. + /// + public MarshalMethodsClassifier? Classifier { get; } + + /// + /// All the Java types discovered in the target architecture's assemblies. + /// + public List AllJavaTypes { get; } + + public List JavaTypesForJCW { get; } + public XAAssemblyResolver Resolver { get; } + public TypeDefinitionCache TypeCache { get; } + public bool JniAddNativeMethodRegistrationAttributePresent { get; set; } + + public NativeCodeGenState (AndroidTargetArch arch, TypeDefinitionCache tdCache, XAAssemblyResolver resolver, List allJavaTypes, List javaTypesForJCW, MarshalMethodsClassifier? classifier) + { + TargetArch = arch; + TypeCache = tdCache; + Resolver = resolver; + AllJavaTypes = allJavaTypes; + JavaTypesForJCW = javaTypesForJCW; + Classifier = classifier; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs index b54208a7d29..a6dbce2777a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -1,13 +1,13 @@ using System; using System.IO; using System.Collections.Generic; -using System.Collections.ObjectModel; using System.Linq; using System.Text; using Microsoft.Build.Utilities; using Java.Interop.Tools.Cecil; +using Microsoft.Build.Utilities; using Mono.Cecil; using Microsoft.Android.Build.Tasks; using Xamarin.Android.Tools; @@ -98,27 +98,13 @@ sealed class ReleaseGenerationState public readonly Dictionary KnownAssemblies; public readonly Dictionary MvidCache; - public readonly IDictionary> TempModules; - - // Just a convenient way to access one of the temp modules dictionaries, to be used when dealing with ABI-agnostic - // types in ProcessReleaseType. - public readonly Dictionary TempModulesAbiAgnostic; + public readonly Dictionary TempModules; - public ReleaseGenerationState (string[] supportedAbis) + public ReleaseGenerationState () { KnownAssemblies = new Dictionary (StringComparer.Ordinal); MvidCache = new Dictionary (); - - var tempModules = new Dictionary> (); - foreach (string abi in supportedAbis) { - var dict = new Dictionary (); - if (TempModulesAbiAgnostic == null) { - TempModulesAbiAgnostic = dict; - } - tempModules.Add (MonoAndroidHelper.AbiToTargetArch (abi), dict); - } - - TempModules = new ReadOnlyDictionary> (tempModules); + TempModules = new Dictionary (); } public void AddKnownAssembly (TypeDefinition td) @@ -135,80 +121,78 @@ public void AddKnownAssembly (TypeDefinition td) public string GetAssemblyName (TypeDefinition td) => td.Module.Assembly.FullName; } - TaskLoggingHelper log; - Encoding outputEncoding; - byte[] moduleMagicString; - byte[] typemapIndexMagicString; - string[] supportedAbis; + readonly Encoding outputEncoding; + readonly byte[] moduleMagicString; + readonly byte[] typemapIndexMagicString; + readonly TaskLoggingHelper log; + readonly NativeCodeGenState state; public IList GeneratedBinaryTypeMaps { get; } = new List (); - public TypeMapGenerator (TaskLoggingHelper log, string[] supportedAbis) + public TypeMapGenerator (TaskLoggingHelper log, NativeCodeGenState state) { this.log = log ?? throw new ArgumentNullException (nameof (log)); - if (supportedAbis == null) - throw new ArgumentNullException (nameof (supportedAbis)); - this.supportedAbis = supportedAbis; - + this.state = state ?? throw new ArgumentNullException (nameof (state)); outputEncoding = Files.UTF8withoutBOM; moduleMagicString = outputEncoding.GetBytes (TypeMapMagicString); typemapIndexMagicString = outputEncoding.GetBytes (TypeMapIndexMagicString); } - void UpdateApplicationConfig (TypeDefinition javaType, ApplicationConfigTaskState appConfState) + void UpdateApplicationConfig (TypeDefinition javaType) { - if (appConfState.JniAddNativeMethodRegistrationAttributePresent) - return; - if (!javaType.HasCustomAttributes) + if (state.JniAddNativeMethodRegistrationAttributePresent || !javaType.HasCustomAttributes) { return; + } foreach (CustomAttribute ca in javaType.CustomAttributes) { - if (!appConfState.JniAddNativeMethodRegistrationAttributePresent && String.Compare ("JniAddNativeMethodRegistrationAttribute", ca.AttributeType.Name, StringComparison.Ordinal) == 0) { - appConfState.JniAddNativeMethodRegistrationAttributePresent = true; + if (!state.JniAddNativeMethodRegistrationAttributePresent && String.Compare ("JniAddNativeMethodRegistrationAttribute", ca.AttributeType.Name, StringComparison.Ordinal) == 0) { + state.JniAddNativeMethodRegistrationAttributePresent = true; break; } } } - public bool Generate (bool debugBuild, bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, bool generateNativeAssembly, out ApplicationConfigTaskState appConfState) + public bool Generate (bool debugBuild, bool skipJniAddNativeMethodRegistrationAttributeScan, string outputDirectory, bool generateNativeAssembly) { - if (String.IsNullOrEmpty (outputDirectory)) + if (String.IsNullOrEmpty (outputDirectory)) { throw new ArgumentException ("must not be null or empty", nameof (outputDirectory)); + } + Directory.CreateDirectory (outputDirectory); - if (!Directory.Exists (outputDirectory)) - Directory.CreateDirectory (outputDirectory); - - appConfState = new ApplicationConfigTaskState { - JniAddNativeMethodRegistrationAttributePresent = skipJniAddNativeMethodRegistrationAttributeScan - }; - + state.JniAddNativeMethodRegistrationAttributePresent = skipJniAddNativeMethodRegistrationAttributeScan; string typemapsOutputDirectory = Path.Combine (outputDirectory, "typemaps"); - if (debugBuild) { - return GenerateDebug (skipJniAddNativeMethodRegistrationAttributeScan, javaTypes, cache, typemapsOutputDirectory, generateNativeAssembly, appConfState); + return GenerateDebug (skipJniAddNativeMethodRegistrationAttributeScan, typemapsOutputDirectory, generateNativeAssembly); } - return GenerateRelease (skipJniAddNativeMethodRegistrationAttributeScan, javaTypes, cache, typemapsOutputDirectory, appConfState); + return GenerateRelease (skipJniAddNativeMethodRegistrationAttributeScan, typemapsOutputDirectory); } - bool GenerateDebug (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, bool generateNativeAssembly, ApplicationConfigTaskState appConfState) + bool GenerateDebug (bool skipJniAddNativeMethodRegistrationAttributeScan, string outputDirectory, bool generateNativeAssembly) { if (generateNativeAssembly) { - return GenerateDebugNativeAssembly (skipJniAddNativeMethodRegistrationAttributeScan, javaTypes, cache, outputDirectory, appConfState); + return GenerateDebugNativeAssembly (skipJniAddNativeMethodRegistrationAttributeScan, outputDirectory); } - return GenerateDebugFiles (skipJniAddNativeMethodRegistrationAttributeScan, javaTypes, cache, outputDirectory, appConfState); + + // Debug builds which don't put typemaps in native assembly must output data files in architecture-specific + // subdirectories, so that fastdev can properly sync them to the device. + // The (empty) native assembly files, however, must still be generated in the usual directory. + return GenerateDebugFiles ( + skipJniAddNativeMethodRegistrationAttributeScan, + typemapFilesOutputDirectory: Path.Combine (outputDirectory, MonoAndroidHelper.ArchToAbi (state.TargetArch)), + llFilesOutputDirectory: outputDirectory + ); } - bool GenerateDebugFiles (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) + bool GenerateDebugFiles (bool skipJniAddNativeMethodRegistrationAttributeScan, string typemapFilesOutputDirectory, string llFilesOutputDirectory) { var modules = new Dictionary (StringComparer.Ordinal); int maxModuleFileNameWidth = 0; int maxModuleNameWidth = 0; var javaDuplicates = new Dictionary> (StringComparer.Ordinal); - foreach (JavaType jt in javaTypes) { - TypeDefinition td = jt.Type; - UpdateApplicationConfig (td, appConfState); + foreach (TypeDefinition td in state.AllJavaTypes) { + UpdateApplicationConfig (td); string moduleName = td.Module.Assembly.Name.Name; ModuleDebugData module; @@ -220,7 +204,7 @@ bool GenerateDebugFiles (bool skipJniAddNativeMethodRegistrationAttributeScan, L ManagedNameWidth = 0, JavaToManagedMap = new List (), ManagedToJavaMap = new List (), - OutputFilePath = Path.Combine (outputDirectory, outputFileName), + OutputFilePath = Path.Combine (typemapFilesOutputDirectory, outputFileName), ModuleName = moduleName, ModuleNameBytes = outputEncoding.GetBytes (moduleName), }; @@ -234,8 +218,8 @@ bool GenerateDebugFiles (bool skipJniAddNativeMethodRegistrationAttributeScan, L modules.Add (moduleName, module); } - TypeMapDebugEntry entry = GetDebugEntry (td, cache); - HandleDebugDuplicates (javaDuplicates, entry, td, cache); + TypeMapDebugEntry entry = GetDebugEntry (td, state.TypeCache); + HandleDebugDuplicates (javaDuplicates, entry, td, state.TypeCache); if (entry.JavaName.Length > module.JavaNameWidth) module.JavaNameWidth = (uint)entry.JavaName.Length + 1; @@ -251,7 +235,7 @@ bool GenerateDebugFiles (bool skipJniAddNativeMethodRegistrationAttributeScan, L PrepareDebugMaps (module); } - string typeMapIndexPath = Path.Combine (outputDirectory, "typemap.index"); + string typeMapIndexPath = Path.Combine (typemapFilesOutputDirectory, "typemap.index"); using (var indexWriter = MemoryStreamPool.Shared.CreateBinaryWriter ()) { OutputModules (modules, indexWriter, maxModuleFileNameWidth + 1); indexWriter.Flush (); @@ -260,23 +244,22 @@ bool GenerateDebugFiles (bool skipJniAddNativeMethodRegistrationAttributeScan, L GeneratedBinaryTypeMaps.Add (typeMapIndexPath); var composer = new TypeMappingDebugNativeAssemblyGenerator (log, new ModuleDebugData ()); - GenerateNativeAssembly (composer, composer.Construct (), outputDirectory); + GenerateNativeAssembly (composer, composer.Construct (), llFilesOutputDirectory); return true; } - bool GenerateDebugNativeAssembly (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) + bool GenerateDebugNativeAssembly (bool skipJniAddNativeMethodRegistrationAttributeScan, string outputDirectory) { var javaToManaged = new List (); var managedToJava = new List (); var javaDuplicates = new Dictionary> (StringComparer.Ordinal); - foreach (JavaType jt in javaTypes) { - TypeDefinition td = jt.Type; - UpdateApplicationConfig (td, appConfState); + foreach (TypeDefinition td in state.AllJavaTypes) { + UpdateApplicationConfig (td); - TypeMapDebugEntry entry = GetDebugEntry (td, cache); - HandleDebugDuplicates (javaDuplicates, entry, td, cache); + TypeMapDebugEntry entry = GetDebugEntry (td, state.TypeCache); + HandleDebugDuplicates (javaDuplicates, entry, td, state.TypeCache); javaToManaged.Add (entry); managedToJava.Add (entry); @@ -377,35 +360,21 @@ string GetManagedTypeName (TypeDefinition td) return $"{managedTypeName}, {td.Module.Assembly.Name.Name}"; } - void ProcessReleaseType (ReleaseGenerationState state, TypeDefinition td, AndroidTargetArch typeArch, ApplicationConfigTaskState appConfState, TypeDefinitionCache cache) + void ProcessReleaseType (ReleaseGenerationState genState, TypeDefinition td) { - UpdateApplicationConfig (td, appConfState); - - state.AddKnownAssembly (td); + UpdateApplicationConfig (td); + genState.AddKnownAssembly (td); // We must NOT use Guid here! The reason is that Guid sort order is different than its corresponding // byte array representation and on the runtime we need the latter in order to be able to binary search // through the module array. byte[] moduleUUID; - if (!state.MvidCache.TryGetValue (td.Module.Mvid, out moduleUUID)) { + if (!genState.MvidCache.TryGetValue (td.Module.Mvid, out moduleUUID)) { moduleUUID = td.Module.Mvid.ToByteArray (); - state.MvidCache.Add (td.Module.Mvid, moduleUUID); - } - - bool abiAgnosticType = typeArch == AndroidTargetArch.None; - Dictionary tempModules; - if (abiAgnosticType) { - tempModules = state.TempModulesAbiAgnostic; - } else { - // It will throw if `typeArch` isn't in the dictionary. This is intentional, since we must have no TypeDefinition entries for architectures not - // mentioned in `supportedAbis`. - try { - tempModules = state.TempModules[typeArch]; - } catch (KeyNotFoundException ex) { - throw new InvalidOperationException ($"Internal error: cannot process type specific to architecture '{typeArch}', since that architecture isn't mentioned in the set of supported ABIs", ex); - } + genState.MvidCache.Add (td.Module.Mvid, moduleUUID); } + Dictionary tempModules = genState.TempModules; if (!tempModules.TryGetValue (moduleUUID, out ModuleReleaseData moduleData)) { moduleData = new ModuleReleaseData { Mvid = td.Module.Mvid, @@ -416,18 +385,10 @@ void ProcessReleaseType (ReleaseGenerationState state, TypeDefinition td, Androi DuplicateTypes = new List (), }; - if (abiAgnosticType) { - // ABI-agnostic types must be added to all the ABIs - foreach (var kvp in state.TempModules) { - kvp.Value.Add (moduleUUID, moduleData); - } - } else { - // ABI-specific types are added only to their respective tempModules - tempModules.Add (moduleUUID, moduleData); - } + tempModules.Add (moduleUUID, moduleData); } - string javaName = Java.Interop.Tools.TypeNameMappings.JavaNativeTypeManager.ToJniName (td, cache); + string javaName = Java.Interop.Tools.TypeNameMappings.JavaNativeTypeManager.ToJniName (td, state.TypeCache); // We will ignore generic types and interfaces when generating the Java to Managed map, but we must not // omit them from the table we output - we need the same number of entries in both java-to-managed and // managed-to-java tables. `SkipInJavaToManaged` set to `true` will cause the native assembly generator @@ -437,7 +398,7 @@ void ProcessReleaseType (ReleaseGenerationState state, TypeDefinition td, Androi JavaName = javaName, ManagedTypeName = td.FullName, Token = td.MetadataToken.ToUInt32 (), - AssemblyNameIndex = state.KnownAssemblies [state.GetAssemblyName (td)], + AssemblyNameIndex = genState.KnownAssemblies [genState.GetAssemblyName (td)], SkipInJavaToManaged = ShouldSkipInJavaToManaged (td), }; @@ -452,42 +413,30 @@ void ProcessReleaseType (ReleaseGenerationState state, TypeDefinition td, Androi } } - bool GenerateRelease (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) + bool GenerateRelease (bool skipJniAddNativeMethodRegistrationAttributeScan, string outputDirectory) { - var state = new ReleaseGenerationState (supportedAbis); - - foreach (JavaType jt in javaTypes) { - if (!jt.IsABiSpecific) { - ProcessReleaseType (state, jt.Type, AndroidTargetArch.None, appConfState, cache); - continue; - } - - foreach (var kvp in jt.PerAbiTypes) { - ProcessReleaseType (state, kvp.Value, kvp.Key, appConfState, cache); - } + var genState = new ReleaseGenerationState (); + foreach (TypeDefinition td in state.AllJavaTypes) { + ProcessReleaseType (genState, td); } - foreach (var kvp in state.TempModules) { - AndroidTargetArch arch = kvp.Key; - Dictionary tempModules = kvp.Value; - ModuleReleaseData[] modules = tempModules.Values.ToArray (); - Array.Sort (modules, new ModuleUUIDArrayComparer ()); - - foreach (ModuleReleaseData module in modules) { - if (module.TypesScratch.Count == 0) { - module.Types = Array.Empty (); - continue; - } + ModuleReleaseData[] modules = genState.TempModules.Values.ToArray (); + Array.Sort (modules, new ModuleUUIDArrayComparer ()); - // No need to sort here, the LLVM IR generator will compute hashes and sort - // the array on write. - module.Types = module.TypesScratch.Values.ToArray (); + foreach (ModuleReleaseData module in modules) { + if (module.TypesScratch.Count == 0) { + module.Types = Array.Empty (); + continue; } - var composer = new TypeMappingReleaseNativeAssemblyGenerator (log, new NativeTypeMappingData (log, modules)); - GenerateNativeAssembly (arch, composer, composer.Construct (), outputDirectory); + // No need to sort here, the LLVM IR generator will compute hashes and sort + // the array on write. + module.Types = module.TypesScratch.Values.ToArray (); } + var composer = new TypeMappingReleaseNativeAssemblyGenerator (log, new NativeTypeMappingData (log, modules)); + GenerateNativeAssembly (composer, composer.Construct (), outputDirectory); + return true; } @@ -496,35 +445,24 @@ bool ShouldSkipInJavaToManaged (TypeDefinition td) return td.IsInterface || td.HasGenericParameters; } - string GetOutputFilePath (string baseFileName, string abi) => $"{baseFileName}.{abi}.ll"; + string GetOutputFilePath (string baseFileName, AndroidTargetArch arch) => $"{baseFileName}.{MonoAndroidHelper.ArchToAbi (arch)}.ll"; - void GenerateNativeAssembly (AndroidTargetArch arch, LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string baseFileName) + void GenerateNativeAssembly (LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string baseFileName) { WriteNativeAssembly ( - arch, composer, typeMapModule, - GetOutputFilePath (baseFileName, ArchToAbi (arch)) + GetOutputFilePath (baseFileName, state.TargetArch) ); } - void GenerateNativeAssembly (LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string baseFileName) - { - foreach (string abi in supportedAbis) { - WriteNativeAssembly ( - GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi), - composer, - typeMapModule, - GetOutputFilePath (baseFileName, abi) - ); - } - } - - void WriteNativeAssembly (AndroidTargetArch arch, LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string outputFile) + void WriteNativeAssembly (LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string outputFile) { + // TODO: each .ll file should have a comment which lists paths to all the DLLs that were used to generate + // the native code using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { try { - composer.Generate (typeMapModule, arch, sw, outputFile); + composer.Generate (typeMapModule, state.TargetArch, sw, outputFile); } catch { throw; } finally { @@ -534,17 +472,6 @@ void WriteNativeAssembly (AndroidTargetArch arch, LLVMIR.LlvmIrComposer composer } } - static string ArchToAbi (AndroidTargetArch arch) - { - return arch switch { - AndroidTargetArch.Arm => "armeabi-v7a", - AndroidTargetArch.Arm64 => "arm64-v8a", - AndroidTargetArch.X86_64 => "x86_64", - AndroidTargetArch.X86 => "x86", - _ => throw new InvalidOperationException ($"Unknown architecture {arch}") - }; - } - // Binary index file format, all data is little-endian: // // [Magic string] # XATI diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs index 61b3f5c314e..eeb9977cd86 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs @@ -1,8 +1,6 @@ using System; using System.Collections; using System.Collections.Generic; -using System.IO.Hashing; -using System.Text; using Microsoft.Build.Utilities; @@ -218,6 +216,7 @@ protected override void Construct (LlvmIrModule module) BeforeWriteCallbackCallerState = cs, GetArrayItemCommentCallback = GetJavaHashesItemComment, GetArrayItemCommentCallbackCallerState = cs, + NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal, }; map_java_hashes.WriteOptions &= ~LlvmIrVariableWriteOptions.ArrayWriteIndexComments; module.Add (map_java_hashes); @@ -264,7 +263,7 @@ uint GetJavaEntryIndex (TypeMapJava javaEntry) throw new InvalidOperationException ("Internal error: construction state expected but not found"); } - return $" {index}: {cs.JavaMap[(int)index].Instance.JavaName}"; + return $" {index} => {cs.JavaMap[(int)index].Instance.JavaName}"; } void GenerateAndSortJavaHashes (LlvmIrVariable variable, LlvmIrModuleTarget target, object? callerState) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs b/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs index 746f45802e7..940b0e7e291 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs @@ -11,105 +11,29 @@ namespace Xamarin.Android.Tasks; class XAAssemblyResolver : IAssemblyResolver { - sealed class CacheEntry : IDisposable - { - bool disposed; - - Dictionary assemblies; - TaskLoggingHelper log; - AndroidTargetArch defaultArch; - - /// - /// This field is to be used by the `Resolve` overloads which don't have a way of indicating the desired ABI target for the assembly, but only when the - /// `AndroidTargetArch.None` entry for the assembly in question is **absent**. The field is always set to some value: either the very first assembly added - /// or the one with the `AndroidTargetArch.None` ABI. The latter always wins. - /// - public AssemblyDefinition Default { get; private set; } - public Dictionary Assemblies => assemblies; - - public CacheEntry (TaskLoggingHelper log, string filePath, AssemblyDefinition asm, AndroidTargetArch arch) - { - if (asm == null) { - throw new ArgumentNullException (nameof (asm)); - } - - this.log = log; - Default = asm; - defaultArch = arch; - assemblies = new Dictionary { - { arch, asm }, - }; - } - - public void Add (AndroidTargetArch arch, AssemblyDefinition asm) - { - if (asm == null) { - throw new ArgumentNullException (nameof (asm)); - } - - if (assemblies.ContainsKey (arch)) { - log.LogWarning ($"Entry for assembly '{asm}', architecture '{arch}' already exists. Replacing the old entry."); - } - - assemblies[arch] = asm; - if (arch == AndroidTargetArch.None && defaultArch != AndroidTargetArch.None) { - Default = asm; - defaultArch = arch; - } - } - - void Dispose (bool disposing) - { - if (disposed || !disposing) { - return; - } - - Default = null; - foreach (var kvp in assemblies) { - kvp.Value?.Dispose (); - } - assemblies.Clear (); - disposed = true; - } - - public void Dispose () - { - Dispose (disposing: true); - GC.SuppressFinalize (this); - } - } - - /// - /// Contains a collection of directories where framework assemblies can be found. This collection **must not** - /// contain any directories which contain ABI-specific assemblies. For those, use - /// - public ICollection FrameworkSearchDirectories { get; } = new List (); - - /// - /// Contains a collection of directories where Xamarin.Android (via linker, for instance) has placed the ABI - /// specific assemblies. Each ABI has its own set of directories to search. - /// - public IDictionary> AbiSearchDirectories { get; } = new Dictionary> (); - readonly List viewStreams = new List (); + readonly Dictionary cache; bool disposed; TaskLoggingHelper log; bool loadDebugSymbols; ReaderParameters readerParameters; - readonly Dictionary cache; + readonly AndroidTargetArch targetArch; + + /// + /// **MUST** point to directories which contain assemblies for single ABI **only**. + /// One special case is when linking isn't enabled, in which instance directories + /// containing ABI-agnostic assemblies can we used as well. + public ICollection SearchDirectories { get; } = new List (); + public AndroidTargetArch TargetArch => targetArch; - public XAAssemblyResolver (TaskLoggingHelper log, bool loadDebugSymbols, ReaderParameters? loadReaderParameters = null) + public XAAssemblyResolver (AndroidTargetArch targetArch, TaskLoggingHelper log, bool loadDebugSymbols, ReaderParameters? loadReaderParameters = null) { + this.targetArch = targetArch; this.log = log; this.loadDebugSymbols = loadDebugSymbols; - this.readerParameters = loadReaderParameters ?? new ReaderParameters(); + this.readerParameters = loadReaderParameters ?? new ReaderParameters (); - cache = new Dictionary (StringComparer.OrdinalIgnoreCase); - } - - public AssemblyDefinition? Resolve (string fullName, ReaderParameters? parameters = null) - { - return Resolve (AssemblyNameReference.Parse (fullName), parameters); + cache = new Dictionary (StringComparer.OrdinalIgnoreCase); } public AssemblyDefinition? Resolve (AssemblyNameReference name) @@ -118,52 +42,38 @@ public XAAssemblyResolver (TaskLoggingHelper log, bool loadDebugSymbols, ReaderP } public AssemblyDefinition? Resolve (AssemblyNameReference name, ReaderParameters? parameters) - { - return Resolve (AndroidTargetArch.None, name, parameters); - } - - public AssemblyDefinition? Resolve (AndroidTargetArch arch, AssemblyNameReference name, ReaderParameters? parameters = null) { string shortName = name.Name; - if (cache.TryGetValue (shortName, out CacheEntry? entry)) { - return SelectAssembly (arch, name.FullName, entry, loading: false); - } - - if (arch == AndroidTargetArch.None) { - return FindAndLoadFromDirectories (arch, FrameworkSearchDirectories, name, parameters); - } - - if (!AbiSearchDirectories.TryGetValue (arch, out ICollection? directories) || directories == null) { - throw CreateLoadException (name); + if (cache.TryGetValue (shortName, out AssemblyDefinition? assembly)) { + return assembly; } - return FindAndLoadFromDirectories (arch, directories, name, parameters); + return FindAndLoadFromDirectories (name, parameters); } - AssemblyDefinition? FindAndLoadFromDirectories (AndroidTargetArch arch, ICollection directories, AssemblyNameReference name, ReaderParameters? parameters) + AssemblyDefinition? FindAndLoadFromDirectories (AssemblyNameReference name, ReaderParameters? parameters) { string? assemblyFile; - foreach (string dir in directories) { + foreach (string dir in SearchDirectories) { if ((assemblyFile = SearchDirectory (name.Name, dir)) != null) { - return Load (arch, assemblyFile, parameters); + return Load (assemblyFile, parameters); } } return null; } - static FileNotFoundException CreateLoadException (AssemblyNameReference name) - { - return new FileNotFoundException ($"Could not load assembly '{name}'."); - } - static string? SearchDirectory (string name, string directory) { if (Path.IsPathRooted (name) && File.Exists (name)) { return name; } - var file = Path.Combine (directory, $"{name}.dll"); + if (!name.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) { + name = $"{name}.dll"; + } + + var file = Path.Combine (directory, name); if (File.Exists (file)) { return file; } @@ -171,12 +81,11 @@ static FileNotFoundException CreateLoadException (AssemblyNameReference name) return null; } - public virtual AssemblyDefinition? Load (AndroidTargetArch arch, string filePath, ReaderParameters? readerParameters = null) + public virtual AssemblyDefinition? Load (string filePath, ReaderParameters? readerParameters = null) { string name = Path.GetFileNameWithoutExtension (filePath); - AssemblyDefinition? assembly; - if (cache.TryGetValue (name, out CacheEntry? entry)) { - assembly = SelectAssembly (arch, name, entry, loading: true); + + if (cache.TryGetValue (name, out AssemblyDefinition? assembly)) { if (assembly != null) { return assembly; } @@ -189,11 +98,8 @@ static FileNotFoundException CreateLoadException (AssemblyNameReference name) return null; } - if (!cache.TryGetValue (name, out entry)) { - entry = new CacheEntry (log, filePath, assembly, arch); - cache.Add (name, entry); - } else { - entry.Add (arch, assembly); + if (!cache.ContainsKey (name)) { + cache.Add (name, assembly); } return assembly; @@ -219,7 +125,7 @@ AssemblyDefinition ReadAssembly (string filePath, ReaderParameters? readerParame try { return LoadFromMemoryMappedFile (filePath, loadReaderParams); } catch (Exception ex) { - log.LogWarning ($"Failed to read '{filePath}' with debugging symbols. Retrying to load it without it. Error details are logged below."); + log.LogWarning ($"Failed to read '{filePath}' with debugging symbols for target architecture '{targetArch}'. Retrying to load it without it. Error details are logged below."); log.LogWarning ($"{ex.ToString ()}"); loadReaderParams.ReadSymbols = false; return LoadFromMemoryMappedFile (filePath, loadReaderParams); @@ -260,47 +166,9 @@ AssemblyDefinition LoadFromMemoryMappedFile (string file, ReaderParameters optio } } - AssemblyDefinition? SelectAssembly (AndroidTargetArch arch, string assemblyName, CacheEntry? entry, bool loading) - { - if (entry == null) { - // Should "never" happen... - throw new ArgumentNullException (nameof (entry)); - } - - if (arch == AndroidTargetArch.None) { - // Disabled for now, generates too much noise. - // if (entry.Assemblies.Count > 1) { - // log.LogWarning ($"Architecture-agnostic entry requested for architecture-specific assembly '{assemblyName}'"); - // } - return entry.Default; - } - - if (!entry.Assemblies.TryGetValue (arch, out AssemblyDefinition? asm)) { - if (loading) { - return null; - } - - if (!entry.Assemblies.TryGetValue (AndroidTargetArch.None, out asm)) { - throw new InvalidOperationException ($"Internal error: assembly '{assemblyName}' for architecture '{arch}' not found in cache entry and architecture-agnostic entry is missing as well"); - } - - if (asm == null) { - throw new InvalidOperationException ($"Internal error: architecture-agnostic cache entry for assembly '{assemblyName}' is null"); - } - - log.LogWarning ($"Returning architecture-agnostic cache entry for assembly '{assemblyName}'. Requested architecture was: {arch}"); - return asm; - } - - if (asm == null) { - throw new InvalidOperationException ($"Internal error: null reference for assembly '{assemblyName}' in assembly cache entry"); - } - - return asm; - } - public void Dispose () { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method Dispose (disposing: true); GC.SuppressFinalize (this); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs index 4b82016eaa9..59cd57ebb53 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs @@ -1,8 +1,9 @@ using System; using System.Collections.Generic; -using System.Collections.ObjectModel; +using System.IO; using Java.Interop.Tools.Cecil; +using Microsoft.Android.Build.Tasks; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using Mono.Cecil; @@ -10,94 +11,80 @@ namespace Xamarin.Android.Tasks; -class JavaType -{ - public readonly TypeDefinition Type; - public readonly IDictionary? PerAbiTypes; - public bool IsABiSpecific { get; } - - public JavaType (TypeDefinition type, IDictionary? perAbiTypes) - { - Type = type; - if (perAbiTypes != null) { - PerAbiTypes = new ReadOnlyDictionary (perAbiTypes); - IsABiSpecific = perAbiTypes.Count > 1 || (perAbiTypes.Count == 1 && !perAbiTypes.ContainsKey (AndroidTargetArch.None)); - } - } -} - class XAJavaTypeScanner { - sealed class TypeData - { - public readonly TypeDefinition FirstType; - public readonly Dictionary PerAbi; - - public bool IsAbiSpecific => !PerAbi.ContainsKey (AndroidTargetArch.None); - - public TypeData (TypeDefinition firstType) - { - FirstType = firstType; - PerAbi = new Dictionary (); - } - } + // Names of assemblies which don't have Mono.Android.dll references, or are framework assemblies, but which must + // be scanned for Java types. + static readonly HashSet SpecialAssemblies = new HashSet (StringComparer.OrdinalIgnoreCase) { + "Mono.Android.dll", + "Mono.Android.Runtime.dll", + }; public bool ErrorOnCustomJavaObject { get; set; } - TaskLoggingHelper log; - TypeDefinitionCache cache; + readonly TaskLoggingHelper log; + readonly TypeDefinitionCache cache; + readonly AndroidTargetArch targetArch; - public XAJavaTypeScanner (TaskLoggingHelper log, TypeDefinitionCache cache) + public XAJavaTypeScanner (AndroidTargetArch targetArch, TaskLoggingHelper log, TypeDefinitionCache cache) { + this.targetArch = targetArch; this.log = log; this.cache = cache; } - public List GetJavaTypes (ICollection inputAssemblies, XAAssemblyResolver resolver) + public List GetJavaTypes (ICollection inputAssemblies, XAAssemblyResolver resolver) { - var types = new Dictionary (StringComparer.Ordinal); + var types = new List (); foreach (ITaskItem asmItem in inputAssemblies) { + if (!ShouldScan (asmItem)) { + log.LogDebugMessage ($"[{targetArch}] Skipping Java type scanning in assembly '{asmItem.ItemSpec}'"); + continue; + } + log.LogDebugMessage ($"[{targetArch}] Scanning assembly '{asmItem.ItemSpec}' for Java types"); + AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (asmItem); - AssemblyDefinition asmdef = resolver.Load (arch, asmItem.ItemSpec); + if (arch != targetArch) { + throw new InvalidOperationException ($"Internal error: assembly '{asmItem.ItemSpec}' should be in the '{targetArch}' architecture, but is in '{arch}' instead."); + } + + AssemblyDefinition asmdef = resolver.Load (asmItem.ItemSpec); foreach (ModuleDefinition md in asmdef.Modules) { foreach (TypeDefinition td in md.Types) { - AddJavaType (td, types, arch); + AddJavaType (td, types); } } } - var ret = new List (); - foreach (var kvp in types) { - ret.Add (new JavaType (kvp.Value.FirstType, kvp.Value.IsAbiSpecific ? kvp.Value.PerAbi : null)); - } - - return ret; + return types; } - void AddJavaType (TypeDefinition type, Dictionary types, AndroidTargetArch arch) + bool ShouldScan (ITaskItem assembly) { - if (type.HasJavaPeer (cache)) { - // For subclasses of e.g. Android.App.Activity. - string typeName = type.GetPartialAssemblyQualifiedName (cache); - if (!types.TryGetValue (typeName, out TypeData typeData)) { - typeData = new TypeData (type); - types.Add (typeName, typeData); - } + string name = Path.GetFileName (assembly.ItemSpec); + if (SpecialAssemblies.Contains (name)) { + return true; + } - if (typeData.PerAbi.ContainsKey (AndroidTargetArch.None)) { - if (arch == AndroidTargetArch.None) { - throw new InvalidOperationException ($"Duplicate type '{type.FullName}' in assembly {type.Module.FileName}"); - } + string? hasMonoAndroidReferenceMetadata = assembly.GetMetadata ("HasMonoAndroidReference"); + if (String.IsNullOrEmpty (hasMonoAndroidReferenceMetadata)) { + return true; // Just in case - the metadata missing might be a false negative + } - throw new InvalidOperationException ($"Previously added type '{type.FullName}' was in ABI-agnostic assembly, new one comes from ABI {arch} assembly"); - } + if (Boolean.TryParse (hasMonoAndroidReferenceMetadata, out bool hasMonoAndroidReference)) { + return hasMonoAndroidReference; + } - if (typeData.PerAbi.ContainsKey (arch)) { - throw new InvalidOperationException ($"Duplicate type '{type.FullName}' in assembly {type.Module.FileName}, for ABI {arch}"); - } + // A catch-all, it's better to do more work than to miss something important. + return true; + } - typeData.PerAbi.Add (arch, type); + void AddJavaType (TypeDefinition type, List types) + { + if (type.HasJavaPeer (cache)) { + // For subclasses of e.g. Android.App.Activity. + types.Add (type); } else if (type.IsClass && !type.IsSubclassOf ("System.Exception", cache) && type.ImplementsInterface ("Android.Runtime.IJavaObject", cache)) { string message = $"XA4212: Type `{type.FullName}` implements `Android.Runtime.IJavaObject` but does not inherit `Java.Lang.Object` or `Java.Lang.Throwable`. This is not supported."; @@ -114,7 +101,7 @@ void AddJavaType (TypeDefinition type, Dictionary types, Andro } foreach (TypeDefinition nested in type.NestedTypes) { - AddJavaType (nested, types, arch); + AddJavaType (nested, types); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 0fda73a444e..71a79a00619 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -94,6 +94,7 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. + - <_AndroidResolvedSatellitePaths Include="@(IntermediateSatelliteAssembliesWithTargetPath->'$(OutDir)%(Culture)\$(TargetName).resources.dll')" /> - + DependsOnTargets="_ResolveAssemblies"> + + + + @@ -2092,6 +2094,7 @@ because xbuild doesn't support framework reference assemblies. AndroidNdkDirectory="$(_AndroidNdkDirectory)" ApkInputPath="$(_PackagedResources)" ApkOutputPath="$(ApkFileIntermediate)" + AppSharedLibrariesDir="$(_AndroidApplicationSharedLibraryPath)" BundleNativeLibraries="$(_BundleResultNativeLibraries)" EmbedAssemblies="$(EmbedAssembliesIntoApk)" ResolvedUserAssemblies="@(_ShrunkUserAssemblies);@(_AndroidResolvedSatellitePaths)" @@ -2128,6 +2131,7 @@ because xbuild doesn't support framework reference assemblies. AndroidNdkDirectory="$(_AndroidNdkDirectory)" ApkInputPath="$(_PackagedResources)" ApkOutputPath="$(_BaseZipIntermediate)" + AppSharedLibrariesDir="$(_AndroidApplicationSharedLibraryPath)" BundleNativeLibraries="$(_BundleResultNativeLibraries)" EmbedAssemblies="$(EmbedAssembliesIntoApk)" ResolvedUserAssemblies="@(_ShrunkUserAssemblies);@(_AndroidResolvedSatellitePaths)" diff --git a/src/monodroid/jni/android-system.cc b/src/monodroid/jni/android-system.cc index 68fdcdb5e16..fce00bae049 100644 --- a/src/monodroid/jni/android-system.cc +++ b/src/monodroid/jni/android-system.cc @@ -254,6 +254,7 @@ AndroidSystem::monodroid_get_system_property_from_overrides ([[maybe_unused]] co return 0; } +// TODO: review this. Do we really have to create the dir in release? void AndroidSystem::create_update_dir (char *override_dir) { diff --git a/src/monodroid/jni/application_dso_stub.cc b/src/monodroid/jni/application_dso_stub.cc index bb9a977e8ce..54eedc4a04c 100644 --- a/src/monodroid/jni/application_dso_stub.cc +++ b/src/monodroid/jni/application_dso_stub.cc @@ -57,7 +57,6 @@ const ApplicationConfig application_config = { .system_property_count = 0, .number_of_assemblies_in_apk = 2, .bundled_assembly_name_width = 0, - .number_of_assembly_store_files = 2, .number_of_dso_cache_entries = 2, .android_runtime_jnienv_class_token = 1, .jnienv_initialize_method_token = 2, @@ -79,7 +78,7 @@ static char second_assembly_name[AssemblyNameWidth]; XamarinAndroidBundledAssembly bundled_assemblies[] = { { - .apk_fd = -1, + .file_fd = -1, .data_offset = 0, .data_size = 0, .data = nullptr, @@ -88,7 +87,7 @@ XamarinAndroidBundledAssembly bundled_assemblies[] = { }, { - .apk_fd = -1, + .file_fd = -1, .data_offset = 0, .data_size = 0, .data = nullptr, @@ -113,18 +112,10 @@ AssemblyStoreSingleAssemblyRuntimeData assembly_store_bundled_assemblies[] = { }, }; -AssemblyStoreRuntimeData assembly_stores[] = { - { - .data_start = nullptr, - .assembly_count = 0, - .assemblies = nullptr, - }, - - { - .data_start = nullptr, - .assembly_count = 0, - .assemblies = nullptr, - }, +AssemblyStoreRuntimeData assembly_store = { + .data_start = nullptr, + .assembly_count = 0, + .assemblies = nullptr, }; constexpr char fake_dso_name[] = "libaot-Some.Assembly.dll.so"; @@ -133,6 +124,7 @@ constexpr char fake_dso_name2[] = "libaot-Another.Assembly.dll.so"; DSOCacheEntry dso_cache[] = { { .hash = xamarin::android::xxhash::hash (fake_dso_name, sizeof(fake_dso_name) - 1), + .real_name_hash = xamarin::android::xxhash::hash (fake_dso_name, sizeof(fake_dso_name) - 1), .ignore = true, .name = fake_dso_name, .handle = nullptr, @@ -140,12 +132,15 @@ DSOCacheEntry dso_cache[] = { { .hash = xamarin::android::xxhash::hash (fake_dso_name2, sizeof(fake_dso_name2) - 1), + .real_name_hash = xamarin::android::xxhash::hash (fake_dso_name2, sizeof(fake_dso_name2) - 1), .ignore = true, .name = fake_dso_name2, .handle = nullptr, }, }; +DSOApkEntry dso_apk_entries[2] {}; + // // Support for marshal methods // diff --git a/src/monodroid/jni/basic-android-system.cc b/src/monodroid/jni/basic-android-system.cc index 5a1fae13dea..e9dbbac2d0e 100644 --- a/src/monodroid/jni/basic-android-system.cc +++ b/src/monodroid/jni/basic-android-system.cc @@ -99,5 +99,11 @@ BasicAndroidSystem::setup_apk_directories (unsigned short running_on_cpu, jstrin char* BasicAndroidSystem::determine_primary_override_dir (jstring_wrapper &home) { - return utils.path_combine (home.get_cstr (), SharedConstants::OVERRIDE_DIRECTORY_NAME); + dynamic_local_string name { home.get_cstr () }; + name.append ("/") + .append (SharedConstants::OVERRIDE_DIRECTORY_NAME) + .append ("/") + .append (SharedConstants::android_lib_abi); + + return utils.strdup_new (name.get ()); } diff --git a/src/monodroid/jni/basic-utilities.cc b/src/monodroid/jni/basic-utilities.cc index a84ec879614..bfe10dfa922 100644 --- a/src/monodroid/jni/basic-utilities.cc +++ b/src/monodroid/jni/basic-utilities.cc @@ -34,7 +34,10 @@ void BasicUtilities::create_public_directory (const char *dir) { mode_t m = umask (0); - mkdir (dir, 0777); + int ret = mkdir (dir, 0777); + if (ret < 0) { + log_warn (LOG_DEFAULT, "Failed to create directory '%s'. %s", dir, std::strerror (errno)); + } umask (m); } diff --git a/src/monodroid/jni/basic-utilities.hh b/src/monodroid/jni/basic-utilities.hh index 3ed41659c91..51a42741ed0 100644 --- a/src/monodroid/jni/basic-utilities.hh +++ b/src/monodroid/jni/basic-utilities.hh @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -13,6 +14,7 @@ #include #include #include +#include #include "java-interop-util.h" #include "helpers.hh" @@ -39,6 +41,27 @@ namespace xamarin::android bool directory_exists (const char *directory); bool file_copy (const char *to, const char *from); + static std::optional get_file_size_at (int dirfd, const char *file_name) noexcept + { + struct stat sbuf; + if (fstatat (dirfd, file_name, &sbuf, 0) == -1) { + log_warn (LOG_ASSEMBLY, "Failed to stat file '%s': %s", file_name, std::strerror (errno)); + return {}; + } + + return static_cast(sbuf.st_size); + } + + static std::optional open_file_ro_at (int dirfd, const char *file_name) noexcept + { + int fd = openat (dirfd, file_name, O_RDONLY); + if (fd < 0) { + log_error (LOG_ASSEMBLY, "Failed to open file '%s' for reading: %s", file_name, std::strerror (errno)); + return {}; + } + + return fd; + } // Make sure that `buf` has enough space! This is by design, the methods are supposed to be fast. template @@ -104,7 +127,7 @@ namespace xamarin::android } template - bool ends_with (internal::dynamic_local_string& str, std::string_view const& sv) const noexcept + bool ends_with (internal::dynamic_local_string const& str, std::string_view const& sv) const noexcept { if (str.length () < sv.length ()) { return false; @@ -113,6 +136,12 @@ namespace xamarin::android return memcmp (str.get () + str.length () - sv.length (), sv.data (), sv.length ()) == 0; } + template + bool ends_with (internal::dynamic_local_string& str, std::string_view const& sv) const noexcept + { + return ends_with(static_cast const&>(str), sv); + } + bool ends_with (const char *str, std::string_view const& sv) const noexcept { size_t len = strlen (str); @@ -190,9 +219,10 @@ namespace xamarin::android return nullptr; } - for (size_t i = str.length () - 1; i >= 0; i--) { - if (str[i] == ch) { - return str.get () + i; + for (size_t i = str.length (); i > 0; i--) { + const size_t index = i - 1; + if (str[index] == ch) { + return str.get () + index; } } @@ -237,6 +267,12 @@ namespace xamarin::android return strdup_new (s, strlen (s)); } + template + char *strdup_new (internal::dynamic_local_string const& buf) noexcept + { + return strdup_new (buf.get (), buf.length ()); + } + char *strdup_new (xamarin::android::internal::string_segment const& s, size_t from_index = 0) noexcept { if (from_index >= s.length ()) { diff --git a/src/monodroid/jni/embedded-assemblies-zip.cc b/src/monodroid/jni/embedded-assemblies-zip.cc index 294c20c49e3..b2631fe8b19 100644 --- a/src/monodroid/jni/embedded-assemblies-zip.cc +++ b/src/monodroid/jni/embedded-assemblies-zip.cc @@ -2,7 +2,6 @@ #include #include #include -#include #include #include @@ -10,17 +9,12 @@ #include "embedded-assemblies.hh" #include "globals.hh" #include "xamarin-app.hh" +#include "xxhash.hh" using namespace xamarin::android::internal; using read_count_type = size_t; -force_inline bool -EmbeddedAssemblies::is_debug_file (dynamic_local_string const& name) noexcept -{ - return utils.ends_with (name, ".pdb"); -} - force_inline bool EmbeddedAssemblies::zip_load_entry_common (size_t entry_index, std::vector const& buf, dynamic_local_string &entry_name, ZipEntryLoadState &state) noexcept { @@ -28,33 +22,38 @@ EmbeddedAssemblies::zip_load_entry_common (size_t entry_index, std::vector const& entry_name, ZipEntryLoadState const& state, [[maybe_unused]] monodroid_should_register should_register) noexcept +{ +#if defined (DEBUG) + const char *last_slash = utils.find_last (entry_name, '/'); + bool entry_is_overridden = last_slash == nullptr ? false : !should_register (last_slash + 1); +#else + constexpr bool entry_is_overridden = false; +#endif + + if (register_debug_symbols && !entry_is_overridden && utils.ends_with (entry_name, SharedConstants::PDB_EXTENSION)) { + if (bundled_debug_data == nullptr) { + bundled_debug_data = new std::vector (); + bundled_debug_data->reserve (application_config.number_of_assemblies_in_apk); + } + + bundled_debug_data->emplace_back (); + set_debug_entry_data (bundled_debug_data->back (), state, entry_name); + return; + } + + if (!utils.ends_with (entry_name, SharedConstants::DLL_EXTENSION)) { + return; + } + +#if defined (DEBUG) + if (entry_is_overridden) { + return; + } +#endif + + if (bundled_assembly_index >= application_config.number_of_assemblies_in_apk || state.bundled_assemblies_slow_path) [[unlikely]] { + if (!state.bundled_assemblies_slow_path && bundled_assembly_index == application_config.number_of_assemblies_in_apk) { + log_warn (LOG_ASSEMBLY, "Number of assemblies stored at build time (%u) was incorrect, switching to slow bundling path.", application_config.number_of_assemblies_in_apk); + } + + if (extra_bundled_assemblies == nullptr) { + extra_bundled_assemblies = new std::vector (); + } + + extra_bundled_assemblies->emplace_back (); + // means we need to allocate memory to store the entry name, only the entries pre-allocated during + // build have valid pointer to the name storage area + set_entry_data (extra_bundled_assemblies->back (), state, entry_name); + return; + } + + log_debug (LOG_ASSEMBLY, "Setting bundled assembly entry data at index %zu", bundled_assembly_index); + set_assembly_entry_data (bundled_assemblies [bundled_assembly_index], state, entry_name); + log_debug (LOG_ASSEMBLY, "[%zu] data set: name == '%s'; file_name == '%s'", bundled_assembly_index, bundled_assemblies [bundled_assembly_index].name, bundled_assemblies [bundled_assembly_index].file_name); + bundled_assembly_index++; + number_of_found_assemblies = bundled_assembly_index; + have_and_want_debug_symbols = register_debug_symbols && bundled_debug_data != nullptr; +} + force_inline void EmbeddedAssemblies::zip_load_individual_assembly_entries (std::vector const& buf, uint32_t num_entries, [[maybe_unused]] monodroid_should_register should_register, ZipEntryLoadState &state) noexcept { + // TODO: do away with all the string manipulation here. Replace it with generating xxhash for the entry name dynamic_local_string entry_name; - bool bundled_assemblies_slow_path = bundled_assembly_index >= application_config.number_of_assemblies_in_apk; - uint32_t max_assembly_name_size = application_config.bundled_assembly_name_width - 1; + configure_state_for_individual_assembly_load (state); // clang-tidy claims we have a leak in the loop: // @@ -90,116 +144,68 @@ EmbeddedAssemblies::zip_load_individual_assembly_entries (std::vector c continue; } -#if defined (DEBUG) - const char *last_slash = utils.find_last (entry_name, '/'); - bool entry_is_overridden = last_slash == nullptr ? false : !should_register (last_slash + 1); -#else - constexpr bool entry_is_overridden = false; -#endif - - if (register_debug_symbols && !entry_is_overridden && is_debug_file (entry_name)) { - if (bundled_debug_data == nullptr) { - bundled_debug_data = new std::vector (); - bundled_debug_data->reserve (application_config.number_of_assemblies_in_apk); - } - - bundled_debug_data->emplace_back (); - set_debug_entry_data (bundled_debug_data->back (), state.apk_fd, state.data_offset, state.file_size, state.prefix_len, max_assembly_name_size, entry_name); - continue; - } - - if (!utils.ends_with (entry_name, SharedConstants::DLL_EXTENSION)) - continue; - -#if defined (DEBUG) - if (entry_is_overridden) - continue; -#endif - - if (bundled_assembly_index >= application_config.number_of_assemblies_in_apk || bundled_assemblies_slow_path) [[unlikely]] { - if (!bundled_assemblies_slow_path && bundled_assembly_index == application_config.number_of_assemblies_in_apk) { - log_warn (LOG_ASSEMBLY, "Number of assemblies stored at build time (%u) was incorrect, switching to slow bundling path."); - } - - if (extra_bundled_assemblies == nullptr) { - extra_bundled_assemblies = new std::vector (); - } - - extra_bundled_assemblies->emplace_back (); - // means we need to allocate memory to store the entry name, only the entries pre-allocated during - // build have valid pointer to the name storage area - set_entry_data (extra_bundled_assemblies->back (), state.apk_fd, state.data_offset, state.file_size, state.prefix_len, max_assembly_name_size, entry_name); - continue; + if (entry_name[state.prefix_len + SharedConstants::REGULAR_ASSEMBLY_MARKER_INDEX] == SharedConstants::REGULAR_ASSEMBLY_MARKER_CHAR) { + unmangle_name (entry_name, state.prefix_len); + } else if (entry_name[state.prefix_len + SharedConstants::SATELLITE_ASSEMBLY_MARKER_INDEX] == SharedConstants::SATELLITE_ASSEMBLY_MARKER_CHAR) { + unmangle_name (entry_name, state.prefix_len); + } else { + continue; // Can't be an assembly, the name's not mangled } - set_assembly_entry_data (bundled_assemblies [bundled_assembly_index], state.apk_fd, state.data_offset, state.file_size, state.prefix_len, max_assembly_name_size, entry_name); - bundled_assembly_index++; - number_of_found_assemblies = bundled_assembly_index; + store_individual_assembly_data (entry_name, state, should_register); } - - have_and_want_debug_symbols = register_debug_symbols && bundled_debug_data != nullptr; } -force_inline void +inline void EmbeddedAssemblies::map_assembly_store (dynamic_local_string const& entry_name, ZipEntryLoadState &state) noexcept { - if (number_of_mapped_assembly_stores >= application_config.number_of_assembly_store_files) { - log_fatal (LOG_ASSEMBLY, "Too many assembly stores. Expected at most %u", application_config.number_of_assembly_store_files); + if (number_of_mapped_assembly_stores > number_of_assembly_store_files) { + log_fatal (LOG_ASSEMBLY, "Too many assembly stores. Expected at most %u", number_of_assembly_store_files); Helpers::abort_application (); } - md_mmap_info assembly_store_map = md_mmap_apk_file (state.apk_fd, state.data_offset, state.file_size, entry_name.get ()); - auto header = static_cast(assembly_store_map.area); + int fd; + bool close_fd; + if (!androidSystem.is_embedded_dso_mode_enabled ()) { + log_debug (LOG_ASSEMBLY, "Mapping assembly blob file from filesystem"); + close_fd = true; - if (header->magic != ASSEMBLY_STORE_MAGIC) { - log_fatal (LOG_ASSEMBLY, "Assembly store '%s' is not a valid Xamarin.Android assembly store file", entry_name.get ()); - Helpers::abort_application (); + // state.file_fd refers to the directory where our files live + auto temp_fd = Util::open_file_ro_at (state.file_fd, entry_name.get ()); + if (!temp_fd) { + return; + } + fd = temp_fd.value (); + } else { + fd = state.file_fd; + close_fd = false; } - if (header->version > ASSEMBLY_STORE_FORMAT_VERSION) { - log_fatal (LOG_ASSEMBLY, "Assembly store '%s' uses format v%u which is not understood by this version of Xamarin.Android", entry_name.get (), header->version); - Helpers::abort_application (); + md_mmap_info assembly_store_map = md_mmap_apk_file (fd, state.data_offset, state.file_size, entry_name.get ()); + if (close_fd) { + close (fd); } + auto header = static_cast(assembly_store_map.area); - if (header->store_id >= application_config.number_of_assembly_store_files) { - log_fatal ( - LOG_ASSEMBLY, - "Assembly store '%s' index %u exceeds the number of stores known at application build time, %u", - entry_name.get (), - header->store_id, - application_config.number_of_assembly_store_files - ); + if (header->magic != ASSEMBLY_STORE_MAGIC) { + log_fatal (LOG_ASSEMBLY, "Assembly store '%s' is not a valid .NET Android assembly store file", entry_name.get ()); Helpers::abort_application (); } - AssemblyStoreRuntimeData &rd = assembly_stores[header->store_id]; - if (rd.data_start != nullptr) { - log_fatal (LOG_ASSEMBLY, "Assembly store '%s' has a duplicate ID (%u)", entry_name.get (), header->store_id); + if (header->version != ASSEMBLY_STORE_FORMAT_VERSION) { + log_fatal (LOG_ASSEMBLY, "Assembly store '%s' uses format version 0x%x, instead of the expected 0x%x", entry_name.get (), header->version, ASSEMBLY_STORE_FORMAT_VERSION); Helpers::abort_application (); } constexpr size_t header_size = sizeof(AssemblyStoreHeader); - rd.data_start = static_cast(assembly_store_map.area); - rd.assembly_count = header->local_entry_count; - rd.assemblies = reinterpret_cast(rd.data_start + header_size); - - number_of_found_assemblies += rd.assembly_count; - - if (header->store_id == 0) { - constexpr size_t bundled_assembly_size = sizeof(AssemblyStoreAssemblyDescriptor); - constexpr size_t hash_entry_size = sizeof(AssemblyStoreHashEntry); - - index_assembly_store_header = header; - - size_t bytes_before_hashes = header_size + (bundled_assembly_size * header->local_entry_count); - if constexpr (std::is_same_v) { - assembly_store_hashes = reinterpret_cast(rd.data_start + bytes_before_hashes + (hash_entry_size * header->global_entry_count)); - } else { - assembly_store_hashes = reinterpret_cast(rd.data_start + bytes_before_hashes); - } - } + assembly_store.data_start = static_cast(assembly_store_map.area); + assembly_store.assembly_count = header->entry_count; + assembly_store.index_entry_count = header->index_entry_count; + assembly_store.assemblies = reinterpret_cast(assembly_store.data_start + header_size + header->index_size); + assembly_store_hashes = reinterpret_cast(assembly_store.data_start + header_size); + number_of_found_assemblies += assembly_store.assembly_count; number_of_mapped_assembly_stores++; have_and_want_debug_symbols = register_debug_symbols; } @@ -212,10 +218,9 @@ EmbeddedAssemblies::zip_load_assembly_store_entries (std::vector const& } dynamic_local_string entry_name; - bool common_assembly_store_found = false; - bool arch_assembly_store_found = false; + bool assembly_store_found = false; - log_debug (LOG_ASSEMBLY, "Looking for assembly stores in APK (common: '%s'; arch-specific: '%s')", assembly_store_common_file_name.data (), assembly_store_arch_file_name.data ()); + log_debug (LOG_ASSEMBLY, "Looking for assembly stores in APK ('%s)", assembly_store_file_path.data ()); for (size_t i = 0; i < num_entries; i++) { if (all_required_zip_entries_found ()) { need_to_scan_more_apks = false; @@ -227,14 +232,30 @@ EmbeddedAssemblies::zip_load_assembly_store_entries (std::vector const& continue; } - if (!common_assembly_store_found && utils.ends_with (entry_name, assembly_store_common_file_name)) { - common_assembly_store_found = true; + if (!assembly_store_found && utils.ends_with (entry_name, assembly_store_file_path)) { + assembly_store_found = true; map_assembly_store (entry_name, state); + continue; } - if (!arch_assembly_store_found && utils.ends_with (entry_name, assembly_store_arch_file_name)) { - arch_assembly_store_found = true; - map_assembly_store (entry_name, state); + if (number_of_zip_dso_entries >= application_config.number_of_shared_libraries) { + continue; + } + + // Since it's not an assembly store, it's a shared library most likely and it is long enough for us not to have + // to check the length + if (utils.ends_with (entry_name, dso_suffix)) { + constexpr size_t apk_lib_prefix_len = apk_lib_prefix.size () - 1; + + const char *const name = entry_name.get () + apk_lib_prefix_len; + DSOApkEntry *apk_entry = reinterpret_cast(reinterpret_cast(dso_apk_entries) + (sizeof(DSOApkEntry) * number_of_zip_dso_entries)); + + apk_entry->name_hash = xxhash::hash (name, entry_name.length () - apk_lib_prefix_len); + apk_entry->offset = state.data_offset; + apk_entry->fd = state.file_fd; + + log_debug (LOG_ASSEMBLY, "Found a shared library entry %s (index: %u; name: %s; hash: 0x%zx; apk offset: %u)", entry_name.get (), number_of_zip_dso_entries, name, apk_entry->name_hash, apk_entry->offset); + number_of_zip_dso_entries++; } } } @@ -262,11 +283,12 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus } std::vector buf (cd_size); + const auto [prefix, prefix_len] = get_assemblies_prefix_and_length (); ZipEntryLoadState state { - .apk_fd = fd, - .apk_name = apk_name, - .prefix = get_assemblies_prefix (), - .prefix_len = get_assemblies_prefix_length (), + .file_fd = fd, + .file_name = apk_name, + .prefix = prefix, + .prefix_len = prefix_len, .buf_offset = 0, .compression_method = 0, .local_header_offset = 0, @@ -289,31 +311,43 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus template force_inline void -EmbeddedAssemblies::set_entry_data (XamarinAndroidBundledAssembly &entry, int apk_fd, uint32_t data_offset, uint32_t data_size, uint32_t prefix_len, uint32_t max_name_size, dynamic_local_string const& entry_name) noexcept +EmbeddedAssemblies::set_entry_data (XamarinAndroidBundledAssembly &entry, ZipEntryLoadState const& state, dynamic_local_string const& entry_name) noexcept { - entry.apk_fd = apk_fd; + entry.file_fd = state.file_fd; if constexpr (NeedsNameAlloc) { - entry.name = utils.strdup_new (entry_name.get () + prefix_len); + entry.name = utils.strdup_new (entry_name.get () + state.prefix_len); + if (!androidSystem.is_embedded_dso_mode_enabled () && state.file_name != nullptr) { + entry.file_name = utils.strdup_new (state.file_name); + } } else { - // entry.name is preallocated on build time here and is max_name_size + 1 bytes long, filled with 0s, thus we + // entry.name is preallocated at build time here and is max_name_size + 1 bytes long, filled with 0s, thus we // don't need to append the terminating NUL even for strings of `max_name_size` characters - strncpy (entry.name, entry_name.get () + prefix_len, max_name_size); + strncpy (entry.name, entry_name.get () + state.prefix_len, state.max_assembly_name_size); + if (!androidSystem.is_embedded_dso_mode_enabled () && state.file_name != nullptr) { + strncpy (entry.file_name, state.file_name, state.max_assembly_file_name_size); + } } - entry.name_length = std::min (static_cast(entry_name.length ()) - prefix_len, max_name_size); - entry.data_offset = data_offset; - entry.data_size = data_size; + entry.name_length = std::min (static_cast(entry_name.length ()) - state.prefix_len, state.max_assembly_name_size); + entry.data_offset = state.data_offset; + entry.data_size = state.file_size; + + log_debug ( + LOG_ASSEMBLY, + "Set bundled assembly entry data. file name: '%s'; entry name: '%s'; data size: %u", + entry.file_name, entry.name, entry.data_size + ); } force_inline void -EmbeddedAssemblies::set_assembly_entry_data (XamarinAndroidBundledAssembly &entry, int apk_fd, uint32_t data_offset, uint32_t data_size, uint32_t prefix_len, uint32_t max_name_size, dynamic_local_string const& entry_name) noexcept +EmbeddedAssemblies::set_assembly_entry_data (XamarinAndroidBundledAssembly &entry, ZipEntryLoadState const& state, dynamic_local_string const& entry_name) noexcept { - set_entry_data (entry, apk_fd, data_offset, data_size, prefix_len, max_name_size, entry_name); + set_entry_data (entry, state, entry_name); } force_inline void -EmbeddedAssemblies::set_debug_entry_data (XamarinAndroidBundledAssembly &entry, int apk_fd, uint32_t data_offset, uint32_t data_size, uint32_t prefix_len, uint32_t max_name_size, dynamic_local_string const& entry_name) noexcept +EmbeddedAssemblies::set_debug_entry_data (XamarinAndroidBundledAssembly &entry, ZipEntryLoadState const& state, dynamic_local_string const& entry_name) noexcept { - set_entry_data (entry, apk_fd, data_offset, data_size, prefix_len, max_name_size, entry_name); + set_entry_data (entry, state, entry_name); } bool diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index f53e696d96a..f9119817818 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -2,15 +2,18 @@ #include #include -#include #include #include #include +#include + #include #include #include #include #include +#include +#include #if defined (HAVE_LZ4) #include @@ -154,7 +157,28 @@ template force_inline void EmbeddedAssemblies::map_runtime_file (XamarinAndroidBundledAssembly& file) noexcept { - md_mmap_info map_info = md_mmap_apk_file (file.apk_fd, file.data_offset, file.data_size, file.name); + int fd; + bool close_fd; + if (!androidSystem.is_embedded_dso_mode_enabled ()) { + log_debug (LOG_ASSEMBLY, "Mapping a runtime file from a filesystem"); + close_fd = true; + + // file.file_fd refers to the directory where our files live + auto temp_fd = Util::open_file_ro_at (file.file_fd, file.file_name); + if (!temp_fd) { + return; + } + fd = temp_fd.value (); + } else { + fd = file.file_fd; + close_fd = false; + } + + md_mmap_info map_info = md_mmap_apk_file (fd, file.data_offset, file.data_size, file.name); + if (close_fd) { + close (fd); + } + if (MonodroidRuntime::is_startup_in_progress ()) { file.data = static_cast(map_info.area); } else { @@ -309,29 +333,14 @@ EmbeddedAssemblies::individual_assemblies_open_from_bundles (dynamic_local_strin return nullptr; } -force_inline const AssemblyStoreHashEntry* -EmbeddedAssemblies::find_assembly_store_entry ([[maybe_unused]] hash_t hash, [[maybe_unused]] const AssemblyStoreHashEntry *entries, [[maybe_unused]] size_t entry_count) noexcept +force_inline const AssemblyStoreIndexEntry* +EmbeddedAssemblies::find_assembly_store_entry (hash_t hash, const AssemblyStoreIndexEntry *entries, size_t entry_count) noexcept { - hash_t entry_hash; - const AssemblyStoreHashEntry *ret = nullptr; - - while (entry_count > 0) { - ret = entries + (entry_count / 2); - if constexpr (std::is_same_v) { - entry_hash = ret->hash64; - } else { - entry_hash = ret->hash32; - } - auto result = hash <=> entry_hash; - - if (result < 0) { - entry_count /= 2; - } else if (result > 0) { - entries = ret + 1; - entry_count -= entry_count / 2 + 1; - } else { - return ret; - } + auto equal = [](AssemblyStoreIndexEntry const& entry, hash_t key) -> bool { return entry.name_hash == key; }; + auto less_than = [](AssemblyStoreIndexEntry const& entry, hash_t key) -> bool { return entry.name_hash < key; }; + ssize_t idx = Search::binary_search (hash, entries, entry_count); + if (idx >= 0) { + return &entries[idx]; } return nullptr; @@ -341,49 +350,30 @@ template force_inline MonoAssembly* EmbeddedAssemblies::assembly_store_open_from_bundles (dynamic_local_string& name, TLoaderData loader_data, bool ref_only) noexcept { - size_t len = name.length (); - bool have_dll_ext = utils.ends_with (name, SharedConstants::DLL_EXTENSION); - - if (have_dll_ext) { - len -= SharedConstants::DLL_EXTENSION.length (); - } - - hash_t name_hash = xxhash::hash (name.get (), len); + hash_t name_hash = xxhash::hash (name.get (), name.length ()); log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: looking for bundled name: '%s' (hash 0x%zx)", name.get (), name_hash); - const AssemblyStoreHashEntry *hash_entry = find_assembly_store_entry (name_hash, assembly_store_hashes, application_config.number_of_assemblies_in_apk); + const AssemblyStoreIndexEntry *hash_entry = find_assembly_store_entry (name_hash, assembly_store_hashes, assembly_store.index_entry_count); if (hash_entry == nullptr) { log_warn (LOG_ASSEMBLY, "Assembly '%s' (hash 0x%zx) not found", name.get (), name_hash); return nullptr; } - if (hash_entry->mapping_index >= application_config.number_of_assemblies_in_apk) { - log_fatal (LOG_ASSEMBLY, "Invalid assembly index %u, exceeds the maximum index of %u", hash_entry->mapping_index, application_config.number_of_assemblies_in_apk - 1); + if (hash_entry->descriptor_index >= assembly_store.assembly_count) { + log_fatal (LOG_ASSEMBLY, "Invalid assembly descriptor index %u, exceeds the maximum value of %u", hash_entry->descriptor_index, assembly_store.assembly_count - 1); Helpers::abort_application (); } - AssemblyStoreSingleAssemblyRuntimeData &assembly_runtime_info = assembly_store_bundled_assemblies[hash_entry->mapping_index]; - if (assembly_runtime_info.image_data == nullptr) { - if (hash_entry->store_id >= application_config.number_of_assembly_store_files) { - log_fatal (LOG_ASSEMBLY, "Invalid assembly store ID %u, exceeds the maximum of %u", hash_entry->store_id, application_config.number_of_assembly_store_files - 1); - Helpers::abort_application (); - } - - AssemblyStoreRuntimeData &rd = assembly_stores[hash_entry->store_id]; - if (hash_entry->local_store_index >= rd.assembly_count) { - log_fatal (LOG_ASSEMBLY, "Invalid index %u into local store assembly descriptor array", hash_entry->local_store_index); - Helpers::abort_application (); - } - - AssemblyStoreAssemblyDescriptor *bba = &rd.assemblies[hash_entry->local_store_index]; + AssemblyStoreEntryDescriptor &store_entry = assembly_store.assemblies[hash_entry->descriptor_index]; + AssemblyStoreSingleAssemblyRuntimeData &assembly_runtime_info = assembly_store_bundled_assemblies[store_entry.mapping_index]; + if (assembly_runtime_info.image_data == nullptr) { // The assignments here don't need to be atomic, the value will always be the same, so even if two threads // arrive here at the same time, nothing bad will happen. - assembly_runtime_info.image_data = rd.data_start + bba->data_offset; - assembly_runtime_info.descriptor = bba; - - if (bba->debug_data_offset != 0) { - assembly_runtime_info.debug_info_data = rd.data_start + bba->debug_data_offset; + assembly_runtime_info.image_data = assembly_store.data_start + store_entry.data_offset; + assembly_runtime_info.descriptor = &store_entry; + if (store_entry.debug_data_offset != 0) { + assembly_runtime_info.debug_info_data = assembly_store.data_start + store_entry.debug_data_offset; } log_debug ( @@ -403,13 +393,6 @@ EmbeddedAssemblies::assembly_store_open_from_bundles (dynamic_local_string (assembly_runtime_info.debug_info_data), static_cast(assembly_runtime_info.descriptor->debug_data_size)); } @@ -457,7 +439,10 @@ EmbeddedAssemblies::open_from_bundles (MonoAssemblyName* aname, TLoaderData load } if (a == nullptr) { - log_warn (LOG_ASSEMBLY, "open_from_bundles: failed to load assembly %s", name.get ()); + log_warn (LOG_ASSEMBLY, "open_from_bundles: failed to load bundled assembly %s", name.get ()); +#if defined(DEBUG) + log_warn (LOG_ASSEMBLY, "open_from_bundles: the assembly might have been uploaded to the device with FastDev instead"); +#endif } return a; @@ -1174,7 +1159,7 @@ EmbeddedAssemblies::try_load_typemaps_from_directory (const char *path) #endif // def DEBUG size_t -EmbeddedAssemblies::register_from (const char *apk_file, monodroid_should_register should_register) +EmbeddedAssemblies::register_from_apk (const char *apk_file, monodroid_should_register should_register) noexcept { size_t prev = number_of_found_assemblies; @@ -1184,3 +1169,211 @@ EmbeddedAssemblies::register_from (const char *apk_file, monodroid_should_regist return number_of_found_assemblies; } + +template +force_inline bool +EmbeddedAssemblies::maybe_register_assembly_from_filesystem ( + [[maybe_unused]] monodroid_should_register should_register, + size_t &assembly_count, + const dirent* dir_entry, + ZipEntryLoadState& state) noexcept +{ + dynamic_local_string entry_name; + auto copy_dentry_and_update_state = [] (dynamic_local_string &name, ZipEntryLoadState& state, const dirent* dir_entry) + { + name.assign_c (dir_entry->d_name); + + // We don't need to duplicate the name here, it will be done farther on + state.file_name = dir_entry->d_name; + }; + + // We check whether dir_entry->d_name is an array with a fixed size and whether it's + // big enough so that we can index the array below without having to worry about buffer + // overflows. These are compile-time checks and the status of the field won't change at + // runtime unless Android breaks compatibility (unlikely). + // + // Currently (Jan 2024), dir_try->d_name is declared as `char[256]` by Bionic + static_assert (std::is_bounded_array_vd_name)>); + static_assert (sizeof(dir_entry->d_name) > SharedConstants::MANGLED_ASSEMBLY_REGULAR_ASSEMBLY_MARKER.size()); + static_assert (sizeof(dir_entry->d_name) > SharedConstants::MANGLED_ASSEMBLY_SATELLITE_ASSEMBLY_MARKER.size()); + + if constexpr (MangledNamesMode) { + // We're only interested in "mangled" file names, namely those starting with either the `lib_` or `lib-` prefixes + if (dir_entry->d_name[SharedConstants::REGULAR_ASSEMBLY_MARKER_INDEX] == SharedConstants::REGULAR_ASSEMBLY_MARKER_CHAR) { + assembly_count++; + copy_dentry_and_update_state (entry_name, state, dir_entry); + unmangle_name (entry_name); + } else if (dir_entry->d_name[SharedConstants::SATELLITE_ASSEMBLY_MARKER_INDEX] == SharedConstants::SATELLITE_ASSEMBLY_MARKER_CHAR) { + assembly_count++; + copy_dentry_and_update_state (entry_name, state, dir_entry); + unmangle_name (entry_name); + } else { + return false; + } + } else { + if (utils.ends_with (dir_entry->d_name, SharedConstants::DLL_EXTENSION) || + utils.ends_with (dir_entry->d_name, SharedConstants::PDB_EXTENSION)) { + assembly_count++; + copy_dentry_and_update_state (entry_name, state, dir_entry); + } else { + return false; + } + + } + state.data_offset = 0; + + auto file_size = Util::get_file_size_at (state.file_fd, state.file_name); + if (!file_size) { + return false; // don't terminate, keep going + } + + state.file_size = static_cast(file_size.value ()); + store_individual_assembly_data (entry_name, state, should_register); + + return false; +} + +force_inline bool +EmbeddedAssemblies::maybe_register_blob_from_filesystem ( + [[maybe_unused]] monodroid_should_register should_register, + size_t &assembly_count, + const dirent* dir_entry, + ZipEntryLoadState& state) noexcept +{ + if (dir_entry->d_name[0] != assembly_store_file_name[0]) { + return false; // keep going + } + + if (strncmp (dir_entry->d_name, assembly_store_file_name.data (), assembly_store_file_name.size ()) != 0) { + return false; // keep going + } + + dynamic_local_string blob_name; + blob_name.assign_c (dir_entry->d_name); + + state.data_offset = 0; + state.file_name = dir_entry->d_name; + + auto file_size = Util::get_file_size_at (state.file_fd, state.file_name); + if (!file_size) { + return false; // don't terminate, keep going + } + state.file_size = static_cast(file_size.value ()); + + map_assembly_store (blob_name, state); + assembly_count = assembly_store.assembly_count; + + return true; +} + +force_inline size_t +EmbeddedAssemblies::register_from_filesystem (const char *lib_dir_path,bool look_for_mangled_names, monodroid_should_register should_register) noexcept +{ + log_debug (LOG_ASSEMBLY, "Looking for assemblies in '%s'", lib_dir_path); + DIR *lib_dir = opendir (lib_dir_path); // TODO: put it in a scope guard at some point + if (lib_dir == nullptr) { + log_warn (LOG_ASSEMBLY, "Unable to open app library directory '%s': %s", lib_dir_path, std::strerror (errno)); + return 0; + } + + ZipEntryLoadState state{}; + configure_state_for_individual_assembly_load (state); + + int dir_fd = dirfd (lib_dir); + if (dir_fd < 0) [[unlikely]] { + log_warn (LOG_ASSEMBLY, "Unable to obtain file descriptor for directory '%s': %s", lib_dir_path, std::strerror (errno)); + closedir (lib_dir); + return 0; + } + + state.file_fd = dup (dir_fd); + if (state.file_fd < 0) [[unlikely]] { + log_warn (LOG_ASSEMBLY, "Unable to duplicate file descriptor %d for directory '%s': %s", dir_fd, lib_dir_path, std::strerror (errno)); + closedir (lib_dir); + return 0; + } + + auto register_fn = + application_config.have_assembly_store ? std::mem_fn (&EmbeddedAssemblies::maybe_register_blob_from_filesystem) : + (look_for_mangled_names ? + std::mem_fn (&EmbeddedAssemblies::maybe_register_assembly_from_filesystem) : + std::mem_fn (&EmbeddedAssemblies::maybe_register_assembly_from_filesystem + ) + ); + + size_t assembly_count = 0; + do { + errno = 0; + dirent *cur = readdir (lib_dir); + if (cur == nullptr) { + if (errno != 0) { + log_warn (LOG_ASSEMBLY, "Failed to open a directory entry from '%s': %s", lib_dir_path, std::strerror (errno)); + continue; // keep going, no harm + } + break; // No more entries, we're done + } + + // We can ignore the obvious entries here... + if (cur->d_name[0] == '.') { + continue; + } + +#if defined (DEBUG) + if (!should_register (cur->d_name)) { + assembly_count++; + continue; + } +#endif // def DEBUG + + // ...and we can handle the runtime config entry + if (!runtime_config_blob_found && std::strncmp (cur->d_name, SharedConstants::RUNTIME_CONFIG_BLOB_NAME.data (), SharedConstants::RUNTIME_CONFIG_BLOB_NAME.size ()) == 0) { + log_debug (LOG_ASSEMBLY, "Mapping runtime config blob from '%s'", cur->d_name); + auto file_size = Util::get_file_size_at (state.file_fd, cur->d_name); + if (!file_size) { + continue; + } + + auto fd = Util::open_file_ro_at (state.file_fd, cur->d_name); + if (!fd) { + continue; + } + + runtime_config_blob_mmap = md_mmap_apk_file (fd.value (), 0, file_size.value (), cur->d_name); + runtime_config_blob_found = true; + continue; + } + + // We get `true` if it's time to terminate + if (register_fn (this, should_register, assembly_count, cur, state)) { + break; + } + } while (true); + closedir (lib_dir); + + return assembly_count; +} + +size_t +EmbeddedAssemblies::register_from_filesystem (monodroid_should_register should_register) noexcept +{ + log_debug (LOG_ASSEMBLY, "Registering assemblies from the filesystem"); + constexpr bool LookForMangledNames = true; + size_t assembly_count = register_from_filesystem ( + androidSystem.app_lib_directories[0], + LookForMangledNames, + should_register + ); + +#if defined(DEBUG) + constexpr bool DoNotLookForMangledNames = false; + + assembly_count += register_from_filesystem ( + androidSystem.get_primary_override_dir (), + DoNotLookForMangledNames, + should_register + ); +#endif + + log_debug (LOG_ASSEMBLY, "Found %zu assemblies on the filesystem", assembly_count); + return assembly_count; +} diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index fc0a0fc2472..72682132caf 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -7,8 +7,10 @@ #include #include #include +#include #include +#include #include #include @@ -58,8 +60,8 @@ namespace xamarin::android::internal { struct ZipEntryLoadState { - int apk_fd; - const char * const apk_name; + int file_fd; + const char * file_name; const char * const prefix; uint32_t prefix_len; size_t buf_offset; @@ -67,6 +69,9 @@ namespace xamarin::android::internal { uint32_t local_header_offset; uint32_t data_offset; uint32_t file_size; + bool bundled_assemblies_slow_path; + uint32_t max_assembly_name_size; + uint32_t max_assembly_file_name_size; }; private: @@ -76,17 +81,26 @@ namespace xamarin::android::internal { static constexpr off_t ZIP_EOCD_LEN = 22; static constexpr off_t ZIP_CENTRAL_LEN = 46; static constexpr off_t ZIP_LOCAL_LEN = 30; - static constexpr std::string_view assemblies_prefix { "assemblies/" }; + static constexpr std::string_view zip_path_separator { "/" }; - static constexpr std::string_view dot { "." }; - static constexpr std::string_view assembly_store_prefix { "assemblies" }; + static constexpr std::string_view apk_lib_dir_name { "lib" }; + static constexpr size_t assemblies_prefix_size = calc_size(apk_lib_dir_name, zip_path_separator, SharedConstants::android_lib_abi, zip_path_separator); + static constexpr auto assemblies_prefix = concat_string_views (apk_lib_dir_name, zip_path_separator, SharedConstants::android_lib_abi, zip_path_separator); + + // We have two records for each assembly, for names with and without the extension + static constexpr uint32_t assembly_store_index_entries_per_assembly = 2; + static constexpr uint32_t number_of_assembly_store_files = 1; + static constexpr std::string_view dso_suffix { ".so" }; + + static constexpr auto apk_lib_prefix = assemblies_prefix; // concat_const (apk_lib_dir_name, zip_path_separator, SharedConstants::android_lib_abi, zip_path_separator); + static constexpr std::string_view assembly_store_prefix { "libassemblies." }; static constexpr std::string_view assembly_store_extension { ".blob" }; - static constexpr size_t assembly_store_common_file_name_size = calc_size (zip_path_separator, assembly_store_prefix, assembly_store_extension); - static constexpr auto assembly_store_common_file_name = concat_string_views (zip_path_separator, assembly_store_prefix, assembly_store_extension); + static constexpr size_t assembly_store_file_name_size = calc_size (assembly_store_prefix, SharedConstants::android_lib_abi, assembly_store_extension, dso_suffix); + static constexpr auto assembly_store_file_name = concat_string_views (assembly_store_prefix, SharedConstants::android_lib_abi, assembly_store_extension, dso_suffix); - static constexpr size_t assembly_store_arch_file_name_size = calc_size (zip_path_separator, assembly_store_prefix, dot, SharedConstants::android_abi, assembly_store_extension); - static constexpr auto assembly_store_arch_file_name = concat_string_views (zip_path_separator, assembly_store_prefix, dot, SharedConstants::android_abi, assembly_store_extension); + static constexpr size_t assembly_store_file_path_size = calc_size(apk_lib_dir_name, zip_path_separator, SharedConstants::android_lib_abi, zip_path_separator, assembly_store_prefix, SharedConstants::android_lib_abi, assembly_store_extension, dso_suffix); + static constexpr auto assembly_store_file_path = concat_string_views (apk_lib_dir_name, zip_path_separator, SharedConstants::android_lib_abi, zip_path_separator, assembly_store_prefix, SharedConstants::android_lib_abi, assembly_store_extension, dso_suffix); public: /* filename is e.g. System.dll, System.dll.mdb, System.pdb */ @@ -109,10 +123,22 @@ namespace xamarin::android::internal { /* returns current number of *all* assemblies found from all invocations */ template - size_t register_from (const char *apk_file) + size_t register_from_apk (const char *apk_file) noexcept { static_assert (should_register_fn != nullptr, "should_register_fn is a required template parameter"); - return register_from (apk_file, should_register_fn); + return register_from_apk (apk_file, should_register_fn); + } + + template + size_t register_from_filesystem () noexcept + { + static_assert (should_register_fn != nullptr, "should_register_fn is a required template parameter"); + return register_from_filesystem (should_register_fn); + } + + static constexpr decltype(assemblies_prefix) const& get_assemblies_prefix () noexcept + { + return assemblies_prefix; } bool get_register_debug_symbols () const @@ -151,13 +177,20 @@ namespace xamarin::android::internal { return; } - abort_unless (index_assembly_store_header != nullptr && assembly_store_hashes != nullptr, "Invalid or incomplete assembly store data"); + abort_unless (assembly_store_hashes != nullptr, "Invalid or incomplete assembly store data"); } private: STATIC_IN_ANDROID_RELEASE const char* typemap_managed_to_java (MonoType *type, MonoClass *klass, const uint8_t *mvid) noexcept; STATIC_IN_ANDROID_RELEASE MonoReflectionType* typemap_java_to_managed (hash_t hash, const MonoString *java_type_name) noexcept; - size_t register_from (const char *apk_file, monodroid_should_register should_register); + size_t register_from_apk (const char *apk_file, monodroid_should_register should_register) noexcept; + size_t register_from_filesystem (monodroid_should_register should_register) noexcept; + size_t register_from_filesystem (const char *dir, bool look_for_mangled_names, monodroid_should_register should_register) noexcept; + + template + bool maybe_register_assembly_from_filesystem (monodroid_should_register should_register, size_t& assembly_count, const dirent* dir_entry, ZipEntryLoadState& state) noexcept; + bool maybe_register_blob_from_filesystem (monodroid_should_register should_register, size_t& assembly_count, const dirent* dir_entry, ZipEntryLoadState& state) noexcept; + void gather_bundled_assemblies_from_apk (const char* apk, monodroid_should_register should_register); template @@ -229,23 +262,24 @@ namespace xamarin::android::internal { bool zip_read_entry_info (std::vector const& buf, dynamic_local_string& file_name, ZipEntryLoadState &state); - const char* get_assemblies_prefix () const + std::tuple get_assemblies_prefix_and_length () const noexcept { - return assemblies_prefix_override != nullptr ? assemblies_prefix_override : assemblies_prefix.data (); - } + if (assemblies_prefix_override != nullptr) { + return { assemblies_prefix_override, static_cast(strlen (assemblies_prefix_override)) }; + } - uint32_t get_assemblies_prefix_length () const noexcept - { - return assemblies_prefix_override != nullptr ? static_cast(strlen (assemblies_prefix_override)) : assemblies_prefix.length (); + if (application_config.have_assembly_store) { + return { apk_lib_prefix.data (), apk_lib_prefix.size () - 1 }; + } + + return {assemblies_prefix.data (), assemblies_prefix.size () - 1}; } bool all_required_zip_entries_found () const noexcept { return - number_of_mapped_assembly_stores == application_config.number_of_assembly_store_files && - ((application_config.have_runtime_config_blob && runtime_config_blob_found) || - !application_config.have_runtime_config_blob) - ; + number_of_mapped_assembly_stores == number_of_assembly_store_files && number_of_zip_dso_entries >= application_config.number_of_shared_libraries + && ((application_config.have_runtime_config_blob && runtime_config_blob_found) || !application_config.have_runtime_config_blob); } static force_inline c_unique_ptr to_utf8 (const MonoString *s) noexcept @@ -253,8 +287,6 @@ namespace xamarin::android::internal { return c_unique_ptr (mono_string_to_utf8 (const_cast(s))); } - bool is_debug_file (dynamic_local_string const& name) noexcept; - template static const Entry* binary_search (const Key *key, const Entry *base, size_t nmemb, size_t extra_size = 0) noexcept; @@ -265,13 +297,75 @@ namespace xamarin::android::internal { static const TypeMapModuleEntry* binary_search (uint32_t key, const TypeMapModuleEntry *arr, uint32_t n) noexcept; #endif template - void set_entry_data (XamarinAndroidBundledAssembly &entry, int apk_fd, uint32_t data_offset, uint32_t data_size, uint32_t prefix_len, uint32_t max_name_size, dynamic_local_string const& entry_name) noexcept; - void set_assembly_entry_data (XamarinAndroidBundledAssembly &entry, int apk_fd, uint32_t data_offset, uint32_t data_size, uint32_t prefix_len, uint32_t max_name_size, dynamic_local_string const& entry_name) noexcept; - void set_debug_entry_data (XamarinAndroidBundledAssembly &entry, int apk_fd, uint32_t data_offset, uint32_t data_size, uint32_t prefix_len, uint32_t max_name_size, dynamic_local_string const& entry_name) noexcept; + void set_entry_data (XamarinAndroidBundledAssembly &entry, ZipEntryLoadState const& state, dynamic_local_string const& entry_name) noexcept; + void set_assembly_entry_data (XamarinAndroidBundledAssembly &entry, ZipEntryLoadState const& state, dynamic_local_string const& entry_name) noexcept; + void set_debug_entry_data (XamarinAndroidBundledAssembly &entry, ZipEntryLoadState const& state, dynamic_local_string const& entry_name) noexcept; void map_assembly_store (dynamic_local_string const& entry_name, ZipEntryLoadState &state) noexcept; - const AssemblyStoreHashEntry* find_assembly_store_entry (hash_t hash, const AssemblyStoreHashEntry *entries, size_t entry_count) noexcept; + const AssemblyStoreIndexEntry* find_assembly_store_entry (hash_t hash, const AssemblyStoreIndexEntry *entries, size_t entry_count) noexcept; + void store_individual_assembly_data (dynamic_local_string const& entry_name, ZipEntryLoadState const& state, monodroid_should_register should_register) noexcept; + + constexpr size_t get_mangled_name_max_size_overhead () + { + return SharedConstants::MANGLED_ASSEMBLY_NAME_EXT.size() + + std::max (SharedConstants::MANGLED_ASSEMBLY_REGULAR_ASSEMBLY_MARKER.size(), SharedConstants::MANGLED_ASSEMBLY_SATELLITE_ASSEMBLY_MARKER.size()) + + 1; // For the extra `-` char in the culture portion of satellite assembly's name + } + + void configure_state_for_individual_assembly_load (ZipEntryLoadState& state) noexcept + { + state.bundled_assemblies_slow_path = bundled_assembly_index >= application_config.number_of_assemblies_in_apk; + state.max_assembly_name_size = application_config.bundled_assembly_name_width - 1; + + // Enough room for the mangle character at the start, plus the extra extension + state.max_assembly_file_name_size = static_cast(state.max_assembly_name_size + get_mangled_name_max_size_overhead ()); + } + + template + static constexpr size_t get_mangled_prefix_length () + { + if constexpr (IsSatelliteAssembly) { + // +1 for the extra `-` char in the culture portion of satellite assembly's name; + return SharedConstants::MANGLED_ASSEMBLY_SATELLITE_ASSEMBLY_MARKER.length () + 1; + } else { + return SharedConstants::MANGLED_ASSEMBLY_REGULAR_ASSEMBLY_MARKER.length (); + } + } + + template + static constexpr size_t get_mangled_data_size () + { + return SharedConstants::MANGLED_ASSEMBLY_NAME_EXT.length () + get_mangled_prefix_length (); + } + + template + static void unmangle_name (dynamic_local_string &name, size_t start_idx = 0) noexcept + { + constexpr size_t mangled_data_size = get_mangled_data_size (); + if (name.length () <= mangled_data_size) { + // Nothing to do, the name is too short + return; + } + + size_t new_size = name.length () - mangled_data_size; + memmove (name.get () + start_idx, name.get () + start_idx + get_mangled_prefix_length (), new_size); + name.set_length (new_size); + + if constexpr (IsSatelliteAssembly) { + // Make sure assembly name is {CULTURE}/assembly.dll + for (size_t idx = start_idx; idx < name.length (); idx++) { + if (name[idx] == SharedConstants::SATELLITE_ASSEMBLY_MARKER_CHAR) { + name[idx] = '/'; + break; + } + } + } + log_debug (LOG_ASSEMBLY, "Unmangled name to '%s'", name.get ()); + }; private: + static inline constexpr bool UnmangleSatelliteAssembly = true; + static inline constexpr bool UnmangleRegularAssembly = false; + std::vector *bundled_debug_data = nullptr; std::vector *extra_bundled_assemblies = nullptr; @@ -291,10 +385,10 @@ namespace xamarin::android::internal { md_mmap_info runtime_config_blob_mmap{}; bool runtime_config_blob_found = false; uint32_t number_of_mapped_assembly_stores = 0; + uint32_t number_of_zip_dso_entries = 0; bool need_to_scan_more_apks = true; - AssemblyStoreHeader *index_assembly_store_header = nullptr; - AssemblyStoreHashEntry *assembly_store_hashes; + AssemblyStoreIndexEntry *assembly_store_hashes; std::mutex assembly_decompress_mutex; }; } diff --git a/src/monodroid/jni/mono-image-loader.hh b/src/monodroid/jni/mono-image-loader.hh index 419cdd41b89..c1dbb913878 100644 --- a/src/monodroid/jni/mono-image-loader.hh +++ b/src/monodroid/jni/mono-image-loader.hh @@ -116,8 +116,8 @@ namespace xamarin::android::internal { #if defined (USE_CACHE) ssize_t index = find_index (hash); if (index < 0) { - log_warn (LOG_ASSEMBLY, "Failed to look up image index for hash 0x%zx", hash); - return image; + log_fatal (LOG_ASSEMBLY, "Failed to look up image index for hash 0x%zx", hash); + Helpers::abort_application (); } // We don't need to worry about locking here. Even if we're overwriting an entry just set by another @@ -130,7 +130,7 @@ namespace xamarin::android::internal { } #if defined (USE_CACHE) - static inline size_t number_of_cache_index_entries = application_config.number_of_assemblies_in_apk * number_of_assembly_name_forms_in_image_cache;; + static inline size_t number_of_cache_index_entries = application_config.number_of_assemblies_in_apk * number_of_assembly_name_forms_in_image_cache; #endif // def USE_CACHE }; } diff --git a/src/monodroid/jni/monodroid-glue.cc b/src/monodroid/jni/monodroid-glue.cc index 60eeeacbab4..1de65c5c46d 100644 --- a/src/monodroid/jni/monodroid-glue.cc +++ b/src/monodroid/jni/monodroid-glue.cc @@ -1,7 +1,6 @@ #include #include #include -#include #include #include #include @@ -13,6 +12,8 @@ #include #include #include + +#include #include #include #include @@ -22,6 +23,7 @@ #include #include +#include #include #include @@ -61,6 +63,7 @@ #include "monovm-properties.hh" #include "startup-aware-lock.hh" #include "timing-internal.hh" +#include "search.hh" //#include "xamarin_getifaddrs.h" @@ -185,16 +188,23 @@ MonodroidRuntime::open_from_update_dir (MonoAssemblyName *aname, [[maybe_unused] fullpath.append (SharedConstants::DLL_EXTENSION); } - log_info (LOG_ASSEMBLY, "open_from_update_dir: trying to open assembly: %s\n", fullpath.get ()); - if (utils.file_exists (fullpath.get ())) - result = mono_assembly_open_full (fullpath.get (), nullptr, 0); + log_debug (LOG_ASSEMBLY, "open_from_update_dir: trying to open assembly: %s\n", fullpath.get ()); + if (utils.file_exists (fullpath.get ())) { + MonoImageOpenStatus status{}; + result = mono_assembly_open_full (fullpath.get (), &status, 0); + if (result == nullptr || status != MonoImageOpenStatus::MONO_IMAGE_OK) { + log_warn (LOG_ASSEMBLY, "Failed to load managed assembly '%s'. %s", fullpath.get (), mono_image_strerror (status)); + } + } else { + log_warn (LOG_ASSEMBLY, "open_from_update_dir: assembly file DOES NOT EXIST"); + } if (result != nullptr) { // TODO: register .mdb, .pdb file break; } } - if (result && utils.should_log (LOG_ASSEMBLY)) { + if (result != nullptr && utils.should_log (LOG_ASSEMBLY)) { log_info_nocheck (LOG_ASSEMBLY, "open_from_update_dir: loaded assembly: %p\n", result); } return result; @@ -221,8 +231,7 @@ MonodroidRuntime::should_register_file ([[maybe_unused]] const char *filename) bool exists = utils.file_exists (p.get ()); if (exists) { - log_info (LOG_ASSEMBLY, "should not register '%s' as it exists in the override directory '%s'", filename, odir); - return !exists; + return false; } } #endif @@ -238,12 +247,21 @@ MonodroidRuntime::gather_bundled_assemblies (jstring_array_wrapper &runtimeApks, if (od == nullptr || !utils.directory_exists (od)) { continue; } - log_info (LOG_ASSEMBLY, "Loading TypeMaps from %s", od); - embeddedAssemblies.try_load_typemaps_from_directory (od); + + // TODO: temporary hack for the location of typemaps, to be fixed + dynamic_local_string above { od }; + above.append ("/.."); + log_debug (LOG_ASSEMBLY, "Loading TypeMaps from %s", above.get()); + embeddedAssemblies.try_load_typemaps_from_directory (above.get()); } } #endif + if (!androidSystem.is_embedded_dso_mode_enabled ()) { + *out_user_assemblies_count = embeddedAssemblies.register_from_filesystem (); + return; + } + int64_t apk_count = static_cast(runtimeApks.get_length ()); size_t prev_num_assemblies = 0; bool got_split_config_abi_apk = false; @@ -255,9 +273,11 @@ MonodroidRuntime::gather_bundled_assemblies (jstring_array_wrapper &runtimeApks, if (have_split_apks) { bool scan_apk = false; + // With split configs we need to scan only the abi apk, because both the assembly stores and the runtime + // configuration blob are in `lib/{ARCH}`, which in turn lives in the split config APK if (!got_split_config_abi_apk && utils.ends_with (apk_file.get_cstr (), SharedConstants::split_config_abi_apk_name)) { got_split_config_abi_apk = scan_apk = true; - } else if (!got_base_apk && utils.ends_with (apk_file.get_cstr (), base_apk_name)) { + } else if (!application_config.have_assembly_store && !got_base_apk && utils.ends_with (apk_file.get_cstr (), base_apk_name)) { got_base_apk = scan_apk = true; } @@ -266,7 +286,7 @@ MonodroidRuntime::gather_bundled_assemblies (jstring_array_wrapper &runtimeApks, } } - size_t cur_num_assemblies = embeddedAssemblies.register_from (apk_file.get_cstr ()); + size_t cur_num_assemblies = embeddedAssemblies.register_from_apk (apk_file.get_cstr ()); *out_user_assemblies_count += (cur_num_assemblies - prev_num_assemblies); prev_num_assemblies = cur_num_assemblies; @@ -623,6 +643,7 @@ MonodroidRuntime::mono_runtime_init ([[maybe_unused]] JNIEnv *env, [[maybe_unuse bool log_methods = FastTiming::enabled () && !FastTiming::is_bare_mode (); if (log_methods) [[unlikely]] { std::unique_ptr jit_log_path {utils.path_combine (AndroidSystem::override_dirs [0], "methods.txt")}; + utils.create_directory (AndroidSystem::override_dirs [0], 0755); jit_log = utils.monodroid_fopen (jit_log_path.get (), "a"); utils.set_world_accessable (jit_log_path.get ()); } @@ -706,19 +727,19 @@ MonodroidRuntime::create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks gather_bundled_assemblies (runtimeApks, &user_assemblies_count, have_split_apks); - size_t blob_time_index; - if (FastTiming::enabled ()) [[unlikely]] { - blob_time_index = internal_timing->start_event (TimingEventKind::RuntimeConfigBlob); - } - if (embeddedAssemblies.have_runtime_config_blob ()) { + size_t blob_time_index; + if (FastTiming::enabled ()) [[unlikely]] { + blob_time_index = internal_timing->start_event (TimingEventKind::RuntimeConfigBlob); + } + runtime_config_args.kind = 1; embeddedAssemblies.get_runtime_config_blob (runtime_config_args.runtimeconfig.data.data, runtime_config_args.runtimeconfig.data.data_len); monovm_runtimeconfig_initialize (&runtime_config_args, cleanup_runtime_config, nullptr); - } - if (FastTiming::enabled ()) [[unlikely]] { - internal_timing->end_event (blob_time_index); + if (FastTiming::enabled ()) [[unlikely]] { + internal_timing->end_event (blob_time_index); + } } if (user_assemblies_count == 0 && androidSystem.count_override_assemblies () == 0 && !is_running_on_desktop) { @@ -727,10 +748,11 @@ MonodroidRuntime::create_domain (JNIEnv *env, jstring_array_wrapper &runtimeApks AndroidSystem::override_dirs [0], (AndroidSystem::override_dirs.size () > 1 && AndroidSystem::override_dirs [1] != nullptr) ? AndroidSystem::override_dirs [1] : ""); #else - log_fatal (LOG_DEFAULT, "No assemblies (or assembly blobs) were found in the application APK file(s)"); + log_fatal (LOG_DEFAULT, "No assemblies (or assembly blobs) were found in the application APK file(s) or on the filesystem"); #endif - log_fatal (LOG_DEFAULT, "Make sure that all entries in the APK directory named `assemblies/` are STORED (not compressed)"); - log_fatal (LOG_DEFAULT, "If Android Gradle Plugin's minification feature is enabled, it is likely all the entries in `assemblies/` are compressed"); + constexpr const char *assemblies_prefix = EmbeddedAssemblies::get_assemblies_prefix ().data (); + log_fatal (LOG_DEFAULT, "Make sure that all entries in the APK directory named `%s` are STORED (not compressed)", assemblies_prefix); + log_fatal (LOG_DEFAULT, "If Android Gradle Plugin's minification feature is enabled, it is likely all the entries in `%s` are compressed", assemblies_prefix); Helpers::abort_application (); } @@ -990,27 +1012,15 @@ MonodroidRuntime::convert_dl_flags (int flags) } force_inline DSOCacheEntry* -MonodroidRuntime::find_dso_cache_entry ([[maybe_unused]] hash_t hash) noexcept +MonodroidRuntime::find_dso_cache_entry (hash_t hash) noexcept { - hash_t entry_hash; - DSOCacheEntry *ret = nullptr; - size_t entry_count = application_config.number_of_dso_cache_entries; - DSOCacheEntry *entries = dso_cache; - - while (entry_count > 0) { - ret = entries + (entry_count / 2); - entry_hash = static_cast (ret->hash); - auto result = hash <=> entry_hash; - - if (result < 0) { - entry_count /= 2; - } else if (result > 0) { - entries = ret + 1; - entry_count -= entry_count / 2 + 1; - } else { - return ret; - } + auto equal = [](DSOCacheEntry const& entry, hash_t key) -> bool { return entry.hash == key; }; + auto less_than = [](DSOCacheEntry const& entry, hash_t key) -> bool { return entry.hash < key; }; + ssize_t idx = Search::binary_search (hash, dso_cache, application_config.number_of_dso_cache_entries); + if (idx >= 0) { + return &dso_cache[idx]; } + return nullptr; } @@ -1018,12 +1028,17 @@ force_inline void* MonodroidRuntime::monodroid_dlopen_log_and_return (void *handle, char **err, const char *full_name, bool free_memory, [[maybe_unused]] bool need_api_init) { if (handle == nullptr && err != nullptr) { - *err = utils.monodroid_strdup_printf ("Could not load library: Library '%s' not found.", full_name); + const char *load_error = dlerror (); + if (load_error == nullptr) { + load_error = "Unknown error"; + } + *err = utils.monodroid_strdup_printf ("Could not load library '%s'. %s", full_name, load_error); } if (free_memory) { delete[] full_name; } + return handle; } @@ -1092,9 +1107,31 @@ MonodroidRuntime::monodroid_dlopen (const char *name, int flags, char **err) noe } StartupAwareLock lock (dso_handle_write_lock); - unsigned int dl_flags = monodroidRuntime.convert_dl_flags (flags); +#if defined (RELEASE) + if (androidSystem.is_embedded_dso_mode_enabled ()) { + DSOApkEntry *apk_entry = dso_apk_entries; + for (size_t i = 0; i < application_config.number_of_shared_libraries; i++) { + if (apk_entry->name_hash != dso->real_name_hash) { + apk_entry++; + continue; + } + + android_dlextinfo dli; + dli.flags = ANDROID_DLEXT_USE_LIBRARY_FD | ANDROID_DLEXT_USE_LIBRARY_FD_OFFSET; + dli.library_fd = apk_entry->fd; + dli.library_fd_offset = apk_entry->offset; + dso->handle = android_dlopen_ext (dso->name, flags, &dli); + if (dso->handle != nullptr) { + return monodroid_dlopen_log_and_return (dso->handle, err, dso->name, false /* name_needs_free */); + } + break; + } + } +#endif + unsigned int dl_flags = monodroidRuntime.convert_dl_flags (flags); dso->handle = androidSystem.load_dso_from_any_directories (dso->name, dl_flags); + if (dso->handle != nullptr) { return monodroid_dlopen_log_and_return (dso->handle, err, dso->name, false /* name_needs_free */); } @@ -1279,6 +1316,9 @@ MonodroidRuntime::set_profile_options () .append (OUTPUT_ARG) .append (output_path.get (), output_path.length ()); } + if (utils.create_directory (AndroidSystem::override_dirs[0], 0) < 0) { + log_warn (LOG_DEFAULT, "Failed to create directory '%s'. %s", AndroidSystem::override_dirs[0], std::strerror (errno)); + } log_warn (LOG_DEFAULT, "Initializing profiler with options: %s", value.get ()); debug.monodroid_profiler_load (androidSystem.get_runtime_libdir (), value.get (), output_path.get ()); diff --git a/src/monodroid/jni/search.hh b/src/monodroid/jni/search.hh index 554126d2bee..be6a24437e4 100644 --- a/src/monodroid/jni/search.hh +++ b/src/monodroid/jni/search.hh @@ -6,26 +6,39 @@ #include "platform-compat.hh" #include "xxhash.hh" +#include "logger.hh" namespace xamarin::android::internal { class Search final { public: - force_inline static ssize_t binary_search (hash_t key, const hash_t *arr, size_t n) noexcept + template + force_inline static ssize_t binary_search (hash_t key, const T *arr, size_t n) noexcept { + static_assert (equal != nullptr, "equal is a required template parameter"); + static_assert (less_than != nullptr, "less_than is a required template parameter"); + ssize_t left = -1; ssize_t right = static_cast(n); while (right - left > 1) { ssize_t middle = (left + right) >> 1; - if (arr[middle] < key) { + if (less_than (arr[middle], key)) { left = middle; } else { right = middle; } } - return arr[right] == key ? right : -1; + return equal (arr[right], key) ? right : -1; + } + + force_inline static ssize_t binary_search (hash_t key, const hash_t *arr, size_t n) noexcept + { + auto equal = [](hash_t const& entry, hash_t key) -> bool { return entry == key; }; + auto less_than = [](hash_t const& entry, hash_t key) -> bool { return entry < key; }; + + return binary_search (key, arr, n); } force_inline static ptrdiff_t binary_search_branchless (hash_t x, const hash_t *arr, uint32_t len) noexcept diff --git a/src/monodroid/jni/shared-constants.hh b/src/monodroid/jni/shared-constants.hh index 10e0c38ec00..2c706ae2df7 100644 --- a/src/monodroid/jni/shared-constants.hh +++ b/src/monodroid/jni/shared-constants.hh @@ -18,6 +18,15 @@ namespace xamarin::android::internal class SharedConstants { public: + // These three MUST be the same as like-named constants in src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Basic.cs + static constexpr std::string_view MANGLED_ASSEMBLY_NAME_EXT { ".so" }; + static constexpr std::string_view MANGLED_ASSEMBLY_REGULAR_ASSEMBLY_MARKER { "lib_" }; + static constexpr size_t REGULAR_ASSEMBLY_MARKER_INDEX = 3; // this ☝️ + static constexpr char REGULAR_ASSEMBLY_MARKER_CHAR = MANGLED_ASSEMBLY_REGULAR_ASSEMBLY_MARKER[REGULAR_ASSEMBLY_MARKER_INDEX]; + static constexpr std::string_view MANGLED_ASSEMBLY_SATELLITE_ASSEMBLY_MARKER { "lib-" }; + static constexpr size_t SATELLITE_ASSEMBLY_MARKER_INDEX = 3; // this ☝️ + static constexpr char SATELLITE_ASSEMBLY_MARKER_CHAR = MANGLED_ASSEMBLY_SATELLITE_ASSEMBLY_MARKER[SATELLITE_ASSEMBLY_MARKER_INDEX]; + static constexpr std::string_view MONO_ANDROID_RUNTIME_ASSEMBLY_NAME { "Mono.Android.Runtime" }; static constexpr std::string_view MONO_ANDROID_ASSEMBLY_NAME { "Mono.Android" }; static constexpr std::string_view JAVA_INTEROP_ASSEMBLY_NAME { "Java.Interop" }; @@ -27,7 +36,12 @@ namespace xamarin::android::internal static constexpr std::string_view ANDROID_ENVIRONMENT_CLASS_NAME { "AndroidEnvironment" }; static constexpr std::string_view ANDROID_RUNTIME_INTERNAL_CLASS_NAME { "AndroidRuntimeInternal" }; static constexpr std::string_view DLL_EXTENSION { ".dll" }; - static constexpr std::string_view RUNTIME_CONFIG_BLOB_NAME { "rc.bin" }; + static constexpr std::string_view PDB_EXTENSION { ".pdb" }; + + static constexpr std::string_view RUNTIME_CONFIG_BLOB_BASE_NAME { "libarc.bin" }; + static constexpr size_t runtime_config_blob_name_size = calc_size (RUNTIME_CONFIG_BLOB_BASE_NAME, MANGLED_ASSEMBLY_NAME_EXT); + static constexpr auto RUNTIME_CONFIG_BLOB_NAME = concat_string_views (RUNTIME_CONFIG_BLOB_BASE_NAME, MANGLED_ASSEMBLY_NAME_EXT); + static constexpr std::string_view MONO_SGEN_SO { "libmonosgen-2.0.so" }; static constexpr std::string_view MONO_SGEN_ARCH_SO { "libmonosgen-" __BITNESS__ "-2.0.so" }; static constexpr std::string_view OVERRIDE_DIRECTORY_NAME { ".__override__" }; diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index 6c5280fec80..7d454880faf 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -2,7 +2,8 @@ #ifndef __XAMARIN_ANDROID_TYPEMAP_H #define __XAMARIN_ANDROID_TYPEMAP_H -#include +#include +#include #include #include @@ -13,8 +14,28 @@ static constexpr uint64_t FORMAT_TAG = 0x00025E6972616D58; // 'Xmari^XY' where XY is the format version static constexpr uint32_t COMPRESSED_DATA_MAGIC = 0x5A4C4158; // 'XALZ', little-endian static constexpr uint32_t ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian -static constexpr uint32_t ASSEMBLY_STORE_FORMAT_VERSION = 1; // Increase whenever an incompatible change is made to the - // assembly store format + +// The highest bit of assembly store version is a 64-bit ABI flag +#if INTPTR_MAX == INT64_MAX +static constexpr uint32_t ASSEMBLY_STORE_64BIT_FLAG = 0x80000000; +#else +static constexpr uint32_t ASSEMBLY_STORE_64BIT_FLAG = 0x00000000; +#endif + +// The second-to-last byte denotes the actual ABI +#if defined(__aarch64__) +static constexpr uint32_t ASSEMBLY_STORE_ABI = 0x00010000; +#elif defined(__arm__) +static constexpr uint32_t ASSEMBLY_STORE_ABI = 0x00020000; +#elif defined(__x86_64__) +static constexpr uint32_t ASSEMBLY_STORE_ABI = 0x00030000; +#elif defined(__i386__) +static constexpr uint32_t ASSEMBLY_STORE_ABI = 0x00040000; +#endif + +// Increase whenever an incompatible change is made to the assembly store format +static constexpr uint32_t ASSEMBLY_STORE_FORMAT_VERSION = 2 | ASSEMBLY_STORE_64BIT_FLAG | ASSEMBLY_STORE_ABI; + static constexpr uint32_t MODULE_MAGIC_NAMES = 0x53544158; // 'XATS', little-endian static constexpr uint32_t MODULE_INDEX_MAGIC = 0x49544158; // 'XATI', little-endian static constexpr uint8_t MODULE_FORMAT_VERSION = 2; // Keep in sync with the value in src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -101,9 +122,10 @@ struct CompressedAssemblies CompressedAssemblyDescriptor *descriptors; }; -struct XamarinAndroidBundledAssembly final +struct XamarinAndroidBundledAssembly { - int32_t apk_fd; + int32_t file_fd; + char *file_name; uint32_t data_offset; uint32_t data_size; uint8_t *data; @@ -114,23 +136,38 @@ struct XamarinAndroidBundledAssembly final // // Assembly store format // -// The separate hash indices for 32 and 64-bit hashes are required because they will be sorted differently. -// The 'index' field of each of the hashes{32,64} entry points not only into the `assemblies` array in the -// store but also into the `uint8_t*` `assembly_store_bundled_assemblies*` arrays. +// Each target ABI/architecture has a single assembly store file, composed of the following parts: +// +// [HEADER] +// [INDEX] +// [ASSEMBLY_DESCRIPTORS] +// [ASSEMBLY DATA] +// +// Formats of the sections above are as follows: // -// This way the `assemblies` array in the store can remain read only, because we write the "mapped" assembly -// pointer somewhere else. Otherwise we'd have to copy the `assemblies` array to a writable area of memory. +// HEADER (fixed size) +// [MAGIC] uint; value: 0x41424158 +// [FORMAT_VERSION] uint; store format version number +// [ENTRY_COUNT] uint; number of entries in the store +// [INDEX_ENTRY_COUNT] uint; number of entries in the index +// [INDEX_SIZE] uint; index size in bytes // -// Each store has a unique ID assigned, which is an index into an array of pointers to arrays which store -// individual assembly addresses. Only store with ID 0 comes with the hashes32 and hashes64 arrays. This is -// done to make it possible to use a single sorted array to find assemblies insted of each store having its -// own sorted array of hashes, which would require several binary searches instead of just one. +// INDEX (variable size, HEADER.ENTRY_COUNT*2 entries, for assembly names with and without the extension) +// [NAME_HASH] uint on 32-bit platforms, ulong on 64-bit platforms; xxhash of the assembly name +// [DESCRIPTOR_INDEX] uint; index into in-store assembly descriptor array // -// AssemblyStoreHeader header; -// AssemblyStoreAssemblyDescriptor assemblies[header.local_entry_count]; -// AssemblyStoreHashEntry hashes32[header.global_entry_count]; // only in assembly store with ID 0 -// AssemblyStoreHashEntry hashes64[header.global_entry_count]; // only in assembly store with ID 0 -// [DATA] +// ASSEMBLY_DESCRIPTORS (variable size, HEADER.ENTRY_COUNT entries), each entry formatted as follows: +// [MAPPING_INDEX] uint; index into a runtime array where assembly data pointers are stored +// [DATA_OFFSET] uint; offset from the beginning of the store to the start of assembly data +// [DATA_SIZE] uint; size of the stored assembly data +// [DEBUG_DATA_OFFSET] uint; offset from the beginning of the store to the start of assembly PDB data, 0 if absent +// [DEBUG_DATA_SIZE] uint; size of the stored assembly PDB data, 0 if absent +// [CONFIG_DATA_OFFSET] uint; offset from the beginning of the store to the start of assembly .config contents, 0 if absent +// [CONFIG_DATA_SIZE] uint; size of the stored assembly .config contents, 0 if absent +// +// ASSEMBLY_NAMES (variable size, HEADER.ENTRY_COUNT entries), each entry formatted as follows: +// [NAME_LENGTH] uint: length of assembly name +// [NAME] byte: UTF-8 bytes of assembly name, without the NUL terminator // // @@ -141,31 +178,21 @@ struct [[gnu::packed]] AssemblyStoreHeader final { uint32_t magic; uint32_t version; - uint32_t local_entry_count; - uint32_t global_entry_count; - uint32_t store_id; + uint32_t entry_count; + uint32_t index_entry_count; + uint32_t index_size; // index size in bytes }; -struct [[gnu::packed]] AssemblyStoreHashEntry final +struct [[gnu::packed]] AssemblyStoreIndexEntry final { - union { - uint64_t hash64; - uint32_t hash32; - }; - - // Index into the array with pointers to assembly data. - // It **must** be unique across all the stores from all the apks - uint32_t mapping_index; - - // Index into the array with assembly descriptors inside a store - uint32_t local_store_index; - - // Index into the array with assembly store mmap addresses - uint32_t store_id; + xamarin::android::hash_t name_hash; + uint32_t descriptor_index; }; -struct [[gnu::packed]] AssemblyStoreAssemblyDescriptor final +struct [[gnu::packed]] AssemblyStoreEntryDescriptor final { + uint32_t mapping_index; + uint32_t data_offset; uint32_t data_size; @@ -180,7 +207,8 @@ struct AssemblyStoreRuntimeData final { uint8_t *data_start; uint32_t assembly_count; - AssemblyStoreAssemblyDescriptor *assemblies; + uint32_t index_entry_count; + AssemblyStoreEntryDescriptor *assemblies; }; struct AssemblyStoreSingleAssemblyRuntimeData final @@ -188,7 +216,7 @@ struct AssemblyStoreSingleAssemblyRuntimeData final uint8_t *image_data; uint8_t *debug_info_data; uint8_t *config_data; - AssemblyStoreAssemblyDescriptor *descriptor; + AssemblyStoreEntryDescriptor *descriptor; }; enum class MonoComponent : uint32_t @@ -217,8 +245,8 @@ struct ApplicationConfig uint32_t system_property_count; uint32_t number_of_assemblies_in_apk; uint32_t bundled_assembly_name_width; - uint32_t number_of_assembly_store_files; uint32_t number_of_dso_cache_entries; + uint32_t number_of_shared_libraries; uint32_t android_runtime_jnienv_class_token; uint32_t jnienv_initialize_method_token; uint32_t jnienv_registerjninatives_method_token; @@ -228,9 +256,17 @@ struct ApplicationConfig const char *android_package_name; }; +struct DSOApkEntry +{ + uint64_t name_hash; + uint32_t offset; // offset into the APK + int32_t fd; // apk file descriptor +}; + struct DSOCacheEntry { uint64_t hash; + uint64_t real_name_hash; bool ignore; const char *name; void *handle; @@ -296,9 +332,10 @@ MONO_API MONO_API_EXPORT const char* const mono_aot_mode_name; MONO_API MONO_API_EXPORT XamarinAndroidBundledAssembly bundled_assemblies[]; MONO_API MONO_API_EXPORT AssemblyStoreSingleAssemblyRuntimeData assembly_store_bundled_assemblies[]; -MONO_API MONO_API_EXPORT AssemblyStoreRuntimeData assembly_stores[]; +MONO_API MONO_API_EXPORT AssemblyStoreRuntimeData assembly_store; MONO_API MONO_API_EXPORT DSOCacheEntry dso_cache[]; +MONO_API MONO_API_EXPORT DSOApkEntry dso_apk_entries[]; // // Support for marshal methods @@ -313,7 +350,7 @@ struct MarshalMethodsManagedClass // Number of assembly name forms for which we generate hashes (essentially file name mutations. For instance // `HelloWorld.dll`, `HelloWorld`, `en-US/HelloWorld` etc). This is multiplied by the number of assemblies in the apk to // obtain number of entries in the `assembly_image_cache_hashes` and `assembly_image_cache_indices` entries -constexpr uint32_t number_of_assembly_name_forms_in_image_cache = 2; +constexpr uint32_t number_of_assembly_name_forms_in_image_cache = 3; // These 3 arrays constitute the cache used to store pointers to loaded managed assemblies. // Three arrays are used so that we can have multiple hashes pointing to the same MonoImage*. diff --git a/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj b/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj index 94a4d58464f..406eb288e40 100644 --- a/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj +++ b/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj @@ -24,7 +24,7 @@ - + diff --git a/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs b/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs index cc83d72b5d3..7f5229a514f 100644 --- a/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs @@ -153,19 +153,6 @@ public void BaseZip () }; string blobEntryPrefix = ArchiveAssemblyHelper.DefaultAssemblyStoreEntryPrefix; - if (usesAssemblyBlobs) { - expectedFiles.Add ($"{blobEntryPrefix}Java.Interop.dll"); - expectedFiles.Add ($"{blobEntryPrefix}Mono.Android.dll"); - expectedFiles.Add ($"{blobEntryPrefix}Localization.dll"); - expectedFiles.Add ($"{blobEntryPrefix}es/Localization.resources.dll"); - expectedFiles.Add ($"{blobEntryPrefix}UnnamedProject.dll"); - } else { - expectedFiles.Add ("root/assemblies/Java.Interop.dll"); - expectedFiles.Add ("root/assemblies/Mono.Android.dll"); - expectedFiles.Add ("root/assemblies/Localization.dll"); - expectedFiles.Add ("root/assemblies/es/Localization.resources.dll"); - expectedFiles.Add ("root/assemblies/UnnamedProject.dll"); - } //These are random files from Google Play Services .aar files expectedFiles.Add ("root/play-services-base.properties"); @@ -174,13 +161,28 @@ public void BaseZip () expectedFiles.Add ("root/play-services-tasks.properties"); foreach (var abi in Abis) { + // All assemblies are in per-abi directories now + if (usesAssemblyBlobs) { + expectedFiles.Add ($"{blobEntryPrefix}{abi}/lib_Java.Interop.dll.so"); + expectedFiles.Add ($"{blobEntryPrefix}{abi}/lib_Mono.Android.dll.so"); + expectedFiles.Add ($"{blobEntryPrefix}{abi}/lib_Localization.dll.so"); + expectedFiles.Add ($"{blobEntryPrefix}{abi}/lib-es-Localization.resources.dll.so"); + expectedFiles.Add ($"{blobEntryPrefix}{abi}/lib_UnnamedProject.dll.so"); + } else { + expectedFiles.Add ($"lib/{abi}/lib_Java.Interop.dll.so"); + expectedFiles.Add ($"lib/{abi}/lib_Mono.Android.dll.so"); + expectedFiles.Add ($"lib/{abi}/lib_Localization.dll.so"); + expectedFiles.Add ($"lib/{abi}/lib-es-Localization.resources.dll.so"); + expectedFiles.Add ($"lib/{abi}/lib_UnnamedProject.dll.so"); + } + expectedFiles.Add ($"lib/{abi}/libmonodroid.so"); expectedFiles.Add ($"lib/{abi}/libmonosgen-2.0.so"); expectedFiles.Add ($"lib/{abi}/libxamarin-app.so"); if (usesAssemblyBlobs) { - expectedFiles.Add ($"{blobEntryPrefix}System.Private.CoreLib.dll"); + expectedFiles.Add ($"{blobEntryPrefix}{abi}/lib_System.Private.CoreLib.dll.so"); } else { - expectedFiles.Add ($"root/assemblies/{abi}/System.Private.CoreLib.dll"); + expectedFiles.Add ($"lib/{abi}/lib_System.Private.CoreLib.dll.so"); } expectedFiles.Add ($"lib/{abi}/libSystem.IO.Compression.Native.so"); expectedFiles.Add ($"lib/{abi}/libSystem.Native.so"); @@ -211,19 +213,6 @@ public void AppBundle () }; string blobEntryPrefix = ArchiveAssemblyHelper.DefaultAssemblyStoreEntryPrefix; - if (usesAssemblyBlobs) { - expectedFiles.Add ($"{blobEntryPrefix}Java.Interop.dll"); - expectedFiles.Add ($"{blobEntryPrefix}Mono.Android.dll"); - expectedFiles.Add ($"{blobEntryPrefix}Localization.dll"); - expectedFiles.Add ($"{blobEntryPrefix}es/Localization.resources.dll"); - expectedFiles.Add ($"{blobEntryPrefix}UnnamedProject.dll"); - } else { - expectedFiles.Add ("base/root/assemblies/Java.Interop.dll"); - expectedFiles.Add ("base/root/assemblies/Mono.Android.dll"); - expectedFiles.Add ("base/root/assemblies/Localization.dll"); - expectedFiles.Add ("base/root/assemblies/es/Localization.resources.dll"); - expectedFiles.Add ("base/root/assemblies/UnnamedProject.dll"); - } //These are random files from Google Play Services .aar files expectedFiles.Add ("base/root/play-services-base.properties"); @@ -232,13 +221,28 @@ public void AppBundle () expectedFiles.Add ("base/root/play-services-tasks.properties"); foreach (var abi in Abis) { + // All assemblies are in per-abi directories now + if (usesAssemblyBlobs) { + expectedFiles.Add ($"{blobEntryPrefix}{abi}/lib_Java.Interop.dll.so"); + expectedFiles.Add ($"{blobEntryPrefix}{abi}/lib_Mono.Android.dll.so"); + expectedFiles.Add ($"{blobEntryPrefix}{abi}/lib_Localization.dll.so"); + expectedFiles.Add ($"{blobEntryPrefix}{abi}/lib-es-Localization.resources.dll.so"); + expectedFiles.Add ($"{blobEntryPrefix}{abi}/lib_UnnamedProject.dll.so"); + } else { + expectedFiles.Add ($"base/lib/{abi}/lib_Java.Interop.dll.so"); + expectedFiles.Add ($"base/lib/{abi}/lib_Mono.Android.dll.so"); + expectedFiles.Add ($"base/lib/{abi}/lib_Localization.dll.so"); + expectedFiles.Add ($"base/lib/{abi}/lib-es-Localization.resources.dll.so"); + expectedFiles.Add ($"base/lib/{abi}/lib_UnnamedProject.dll.so"); + } + expectedFiles.Add ($"base/lib/{abi}/libmonodroid.so"); expectedFiles.Add ($"base/lib/{abi}/libmonosgen-2.0.so"); expectedFiles.Add ($"base/lib/{abi}/libxamarin-app.so"); if (usesAssemblyBlobs) { - expectedFiles.Add ($"{blobEntryPrefix}System.Private.CoreLib.dll"); + expectedFiles.Add ($"{blobEntryPrefix}{abi}/lib_System.Private.CoreLib.dll.so"); } else { - expectedFiles.Add ($"base/root/assemblies/{abi}/System.Private.CoreLib.dll"); + expectedFiles.Add ($"base/lib/{abi}/lib_System.Private.CoreLib.dll.so"); } expectedFiles.Add ($"base/lib/{abi}/libSystem.IO.Compression.Native.so"); expectedFiles.Add ($"base/lib/{abi}/libSystem.Native.so"); diff --git a/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs b/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs index f25d948b44c..20f9de4f0e6 100644 --- a/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs @@ -15,12 +15,12 @@ namespace Xamarin.Android.Build.Tests [Category ("UsesDevice")] public class InstallTests : DeviceTest { - string GetContentFromAllOverrideDirectories (string packageName, bool useRunAsCommand = true) + string GetContentFromAllOverrideDirectories (string packageName, string abi, bool useRunAsCommand = true) { var adbShellArgs = $"shell run-as {packageName} ls"; var directorylist = string.Empty; - foreach (var dir in GetOverrideDirectoryPaths (packageName)) { + foreach (var dir in GetOverrideDirectoryPaths (packageName, abi)) { var listing = RunAdbCommand ($"{adbShellArgs} {dir}"); if (!listing.Contains ("No such file or directory") && !listing.Contains ("Permission denied")) directorylist += $"{listing} "; @@ -75,7 +75,7 @@ public void InstallAndUnInstall ([Values (false, true)] bool isRelease) Assert.AreEqual ($"package:{proj.PackageName}", RunAdbCommand ($"shell pm list packages {proj.PackageName}").Trim (), $"{proj.PackageName} is not installed on the device."); - var directorylist = GetContentFromAllOverrideDirectories (proj.PackageName); + var directorylist = GetContentFromAllOverrideDirectories (proj.PackageName, DeviceAbi); if (!isRelease) { StringAssert.Contains ($"{proj.AssemblyName}", directorylist, $"{proj.AssemblyName} not found in fastdev directory."); } @@ -136,7 +136,7 @@ public void SwitchConfigurationsShouldRedeploy () Assert.AreEqual ($"package:{proj.PackageName}", RunAdbCommand ($"shell pm list packages {proj.PackageName}").Trim (), $"{proj.PackageName} is not installed on the device."); - var directorylist = GetContentFromAllOverrideDirectories (proj.PackageName); + var directorylist = GetContentFromAllOverrideDirectories (proj.PackageName, DeviceAbi); StringAssert.Contains ($"{proj.AssemblyName}", directorylist, $"{proj.AssemblyName} not found in fastdev directory."); proj.IsRelease = true; @@ -145,7 +145,7 @@ public void SwitchConfigurationsShouldRedeploy () Assert.AreEqual ($"package:{proj.PackageName}", RunAdbCommand ($"shell pm list packages {proj.PackageName}").Trim (), $"{proj.PackageName} is not installed on the device."); - directorylist = GetContentFromAllOverrideDirectories (proj.PackageName); + directorylist = GetContentFromAllOverrideDirectories (proj.PackageName, DeviceAbi); Assert.AreEqual ("", directorylist.Trim (), "fastdev directory should NOT exist for Release builds."); proj.IsRelease = false; @@ -154,7 +154,7 @@ public void SwitchConfigurationsShouldRedeploy () Assert.AreEqual ($"package:{proj.PackageName}", RunAdbCommand ($"shell pm list packages {proj.PackageName}").Trim (), $"{proj.PackageName} is not installed on the device."); - directorylist = GetContentFromAllOverrideDirectories (proj.PackageName); + directorylist = GetContentFromAllOverrideDirectories (proj.PackageName, DeviceAbi); StringAssert.Contains ($"{proj.AssemblyName}", directorylist, $"{proj.AssemblyName} not found in fastdev directory."); Assert.IsTrue (builder.Uninstall (proj)); @@ -178,7 +178,7 @@ public void InstallWithoutSharedRuntime () proj.SetProperty (proj.ReleaseProperties, "AndroidPackageFormat", "apk"); var abis = new [] { "armeabi-v7a", "arm64-v8a", "x86", "x86_64" }; - proj.SetAndroidSupportedAbis (abis); + proj.SetRuntimeIdentifiers (abis); using (var builder = CreateApkBuilder ()) { if (RunAdbCommand ("shell pm list packages Mono.Android.DebugRuntime").Trim ().Length != 0) RunAdbCommand ("uninstall Mono.Android.DebugRuntime"); @@ -200,10 +200,10 @@ public void InstallWithoutSharedRuntime () //FIXME: https://github.com/xamarin/androidtools/issues/141 //Assert.AreEqual (0, RunAdbCommand ("shell pm list packages Mono.Android.DebugRuntime").Trim ().Length, // "The Shared Runtime should not have been installed."); - var directorylist = GetContentFromAllOverrideDirectories (proj.PackageName); - StringAssert.Contains ($"{proj.ProjectName}.dll", directorylist, $"{proj.ProjectName}.dll should exist in the .__override__ directory."); - StringAssert.Contains ($"System.Private.CoreLib.dll", directorylist, $"System.Private.CoreLib.dll should exist in the .__override__ directory."); - StringAssert.Contains ($"Mono.Android.dll", directorylist, $"Mono.Android.dll should exist in the .__override__ directory."); + var directorylist = GetContentFromAllOverrideDirectories (proj.PackageName, DeviceAbi); + StringAssert.Contains ($"{proj.ProjectName}.dll", directorylist, $"{proj.ProjectName}.dll should exist in the .__override__/{DeviceAbi} directory."); + StringAssert.Contains ($"System.Private.CoreLib.dll", directorylist, $"System.Private.CoreLib.dll should exist in the .__override__/{DeviceAbi} directory."); + StringAssert.Contains ($"Mono.Android.dll", directorylist, $"Mono.Android.dll should exist in the .__override__/{DeviceAbi} directory."); Assert.IsTrue (builder.Uninstall (proj), "unnstall should have succeeded."); } } @@ -249,7 +249,7 @@ public void ToggleFastDev () using (var builder = CreateApkBuilder (Path.Combine ("temp", TestContext.CurrentContext.Test.Name))) { Assert.IsTrue (builder.Install (proj), "Install should have succeeded."); - var directorylist = GetContentFromAllOverrideDirectories (proj.PackageName); + var directorylist = GetContentFromAllOverrideDirectories (proj.PackageName, DeviceAbi); StringAssert.Contains ($"{proj.ProjectName}.dll", directorylist, $"{proj.ProjectName}.dll should exist in the .__override__ directory."); //Now toggle FastDev to OFF @@ -258,7 +258,7 @@ public void ToggleFastDev () Assert.IsTrue (builder.Install (proj), "Second install should have succeeded."); - directorylist = GetContentFromAllOverrideDirectories (proj.PackageName); + directorylist = GetContentFromAllOverrideDirectories (proj.PackageName, DeviceAbi); Assert.AreEqual ("", directorylist, "There should be no files in Fast Dev directories! Instead found: " + directorylist); //Deploy one last time to verify install still works without the .__override__ directory existing @@ -331,7 +331,7 @@ public void LoggingPropsShouldCreateOverrideDirForRelease () var didLaunch = WaitForActivityToStart (proj.PackageName, "MainActivity", Path.Combine (Root, builder.ProjectDirectory, "logcat.log"), 30); ClearShellProp ("debug.mono.log"); Assert.True (didLaunch, "Activity should have started."); - var directorylist = GetContentFromAllOverrideDirectories (proj.PackageName); + var directorylist = GetContentFromAllOverrideDirectories (proj.PackageName, DeviceAbi); builder.Uninstall (proj); StringAssert.Contains ("methods.txt", directorylist, $"methods.txt did not exist in the .__override__ directory.\nFound:{directorylist}"); } @@ -480,9 +480,10 @@ public void LocalizedAssemblies_ShouldBeFastDeployed () .Select (r => r = r.Replace (projectOutputPath, string.Empty).Replace ("\\", "/")); var overrideContents = string.Empty; - foreach (var dir in GetOverrideDirectoryPaths (app.PackageName)) { + foreach (var dir in GetOverrideDirectoryPaths (app.PackageName, DeviceAbi)) { overrideContents += RunAdbCommand ($"shell run-as {app.PackageName} find {dir}"); } + Console.WriteLine ($"#grendel: overrideContents == {overrideContents}"); Assert.IsTrue (resourceFilesFromDisk.Any (), $"Unable to find any localized assemblies in {resourceFilesFromDisk}"); foreach (var res in resourceFilesFromDisk) { StringAssert.Contains (res, overrideContents, $"{res} did not exist in the .__override__ directory.\nFound:{overrideContents}"); diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreExplorer.cs b/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreExplorer.cs new file mode 100644 index 00000000000..c99a23d0636 --- /dev/null +++ b/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreExplorer.cs @@ -0,0 +1,209 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Xamarin.Android.Tools; +using Xamarin.Tools.Zip; + +namespace Xamarin.Android.AssemblyStore; + +class AssemblyStoreExplorer +{ + readonly AssemblyStoreReader reader; + + public string StorePath { get; } + public AndroidTargetArch? TargetArch { get; } + public uint AssemblyCount { get; } + public uint IndexEntryCount { get; } + public IList? Assemblies { get; } + public IDictionary? AssembliesByName { get; } + public bool Is64Bit { get; } + + protected AssemblyStoreExplorer (Stream storeStream, string path) + { + StorePath = path; + var storeReader = AssemblyStoreReader.Create (storeStream, path); + if (storeReader == null) { + storeStream.Dispose (); + throw new NotSupportedException ($"Format of assembly store '{path}' is unsupported"); + } + + reader = storeReader; + TargetArch = reader.TargetArch; + AssemblyCount = reader.AssemblyCount; + IndexEntryCount = reader.IndexEntryCount; + Assemblies = reader.Assemblies; + Is64Bit = reader.Is64Bit; + + var dict = new Dictionary (StringComparer.Ordinal); + foreach (AssemblyStoreItem item in Assemblies) { + dict.Add (item.Name, item); + } + AssembliesByName = dict.AsReadOnly (); + } + + protected AssemblyStoreExplorer (FileInfo storeInfo) + : this (storeInfo.OpenRead (), storeInfo.FullName) + {} + + public static (IList? explorers, string? errorMessage) Open (string inputFile) + { + (FileFormat format, FileInfo? info) = Utils.DetectFileFormat (inputFile); + if (info == null) { + return (null, $"File '{inputFile}' does not exist."); + } + + switch (format) { + case FileFormat.Unknown: + return (null, $"File '{inputFile}' has an unknown format."); + + case FileFormat.Zip: + return (null, $"File '{inputFile}' is a ZIP archive, but not an Android one."); + + case FileFormat.AssemblyStore: + return (new List { new AssemblyStoreExplorer (info)}, null); + + case FileFormat.Aab: + return OpenAab (info); + + case FileFormat.AabBase: + return OpenAabBase (info); + + case FileFormat.Apk: + return OpenApk (info); + + default: + return (null, $"File '{inputFile}' has an unsupported format '{format}'"); + } + } + + static (IList? explorers, string? errorMessage) OpenAab (FileInfo fi) + { + return OpenCommon ( + fi, + new List> { + StoreReader_V2.AabPaths, + StoreReader_V1.AabPaths, + } + ); + } + + static (IList? explorers, string? errorMessage) OpenAabBase (FileInfo fi) + { + return OpenCommon ( + fi, + new List> { + StoreReader_V2.AabBasePaths, + StoreReader_V1.AabBasePaths, + } + ); + } + + static (IList? explorers, string? errorMessage) OpenApk (FileInfo fi) + { + return OpenCommon ( + fi, + new List> { + StoreReader_V2.ApkPaths, + StoreReader_V1.ApkPaths, + } + ); + } + + static (IList? explorers, string? errorMessage) OpenCommon (FileInfo fi, List> pathLists) + { + using var zip = ZipArchive.Open (fi.FullName, FileMode.Open); + IList? explorers; + string? errorMessage; + bool pathsFound; + + foreach (IList paths in pathLists) { + (explorers, errorMessage, pathsFound) = TryLoad (fi, zip, paths); + if (pathsFound) { + return (explorers, errorMessage); + } + } + + return (null, "Unable to find any blob entries"); + } + + static (IList? explorers, string? errorMessage, bool pathsFound) TryLoad (FileInfo fi, ZipArchive zip, IList paths) + { + var ret = new List (); + + foreach (string path in paths) { + if (!zip.ContainsEntry (path)) { + continue; + } + + ZipEntry entry = zip.ReadEntry (path); + var stream = new MemoryStream (); + entry.Extract (stream); + ret.Add (new AssemblyStoreExplorer (stream, $"{fi.FullName}!{path}")); + } + + if (ret.Count == 0) { + return (null, null, false); + } + + return (ret, null, true); + } + + public Stream? ReadImageData (AssemblyStoreItem item, bool uncompressIfNeeded = false) + { + return reader.ReadEntryImageData (item, uncompressIfNeeded); + } + + string EnsureCorrectAssemblyName (string assemblyName) + { + assemblyName = Path.GetFileName (assemblyName); + if (reader.NeedsExtensionInName) { + if (!assemblyName.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) { + return $"{assemblyName}.dll"; + } + } else { + if (assemblyName.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) { + return Path.GetFileNameWithoutExtension (assemblyName); + } + } + + return assemblyName; + } + + public IList? Find (string assemblyName, AndroidTargetArch? targetArch = null) + { + if (Assemblies == null) { + return null; + } + + assemblyName = EnsureCorrectAssemblyName (assemblyName); + var items = new List (); + foreach (AssemblyStoreItem item in Assemblies) { + if (String.CompareOrdinal (assemblyName, item.Name) != 0) { + continue; + } + + if (targetArch != null && item.TargetArch != targetArch) { + continue; + } + + items.Add (item); + } + + if (items.Count == 0) { + return null; + } + + return items; + } + + public bool Contains (string assemblyName, AndroidTargetArch? targetArch = null) + { + IList? items = Find (assemblyName, targetArch); + if (items == null || items.Count == 0) { + return false; + } + + return true; + } +} diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreItem.cs b/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreItem.cs new file mode 100644 index 00000000000..d2ee02cf0a4 --- /dev/null +++ b/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreItem.cs @@ -0,0 +1,26 @@ +using System.Collections.Generic; + +using Xamarin.Android.Tools; + +namespace Xamarin.Android.AssemblyStore; + +abstract class AssemblyStoreItem +{ + public string Name { get; } + public IList Hashes { get; } + public bool Is64Bit { get; } + public uint DataOffset { get; protected set; } + public uint DataSize { get; protected set; } + public uint DebugOffset { get; protected set; } + public uint DebugSize { get; protected set; } + public uint ConfigOffset { get; protected set; } + public uint ConfigSize { get; protected set; } + public AndroidTargetArch TargetArch { get; protected set; } + + protected AssemblyStoreItem (string name, bool is64Bit, List hashes) + { + Name = name; + Hashes = hashes.AsReadOnly (); + Is64Bit = is64Bit; + } +} diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreReader.cs b/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreReader.cs new file mode 100644 index 00000000000..cc39baa6ecc --- /dev/null +++ b/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreReader.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +using Xamarin.Android.Tools; + +namespace Xamarin.Android.AssemblyStore; + +abstract class AssemblyStoreReader +{ + static readonly UTF8Encoding ReaderEncoding = new UTF8Encoding (false); + + protected Stream StoreStream { get; } + + public abstract string Description { get; } + public abstract bool NeedsExtensionInName { get; } + public string StorePath { get; } + + public AndroidTargetArch TargetArch { get; protected set; } = AndroidTargetArch.Arm; + public uint AssemblyCount { get; protected set; } + public uint IndexEntryCount { get; protected set; } + public IList? Assemblies { get; protected set; } + public bool Is64Bit { get; protected set; } + + protected AssemblyStoreReader (Stream store, string path) + { + StoreStream = store; + StorePath = path; + } + + public static AssemblyStoreReader? Create (Stream store, string path) + { + AssemblyStoreReader? reader = MakeReaderReady (new StoreReader_V1 (store, path)); + if (reader != null) { + return reader; + } + + reader = MakeReaderReady (new StoreReader_V2 (store, path)); + if (reader != null) { + return reader; + } + + return null; + } + + static AssemblyStoreReader? MakeReaderReady (AssemblyStoreReader reader) + { + if (!reader.IsSupported ()) { + return null; + } + + reader.Prepare (); + return reader; + } + + protected BinaryReader CreateReader () => new BinaryReader (StoreStream, ReaderEncoding, leaveOpen: true); + + protected abstract bool IsSupported (); + protected abstract void Prepare (); + + public Stream ReadEntryImageData (AssemblyStoreItem entry, bool uncompressIfNeeded = false) + { + StoreStream.Seek (entry.DataOffset, SeekOrigin.Begin); + var stream = new MemoryStream (); + + if (uncompressIfNeeded) { + throw new NotImplementedException (); + } + + const long BufferSize = 65535; + byte[] buffer = Utils.BytePool.Rent ((int)BufferSize); + long remainingToRead = entry.DataSize; + + while (remainingToRead > 0) { + int nread = StoreStream.Read (buffer, 0, (int)Math.Min (BufferSize, remainingToRead)); + stream.Write (buffer, 0, nread); + remainingToRead -= (long)nread; + } + stream.Flush (); + + return stream; + } +} diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/FileFormat.cs b/tools/assembly-store-reader-mk2/AssemblyStore/FileFormat.cs new file mode 100644 index 00000000000..4a02e0ae0c3 --- /dev/null +++ b/tools/assembly-store-reader-mk2/AssemblyStore/FileFormat.cs @@ -0,0 +1,11 @@ +namespace Xamarin.Android.AssemblyStore; + +enum FileFormat +{ + Aab, + AabBase, + Apk, + AssemblyStore, + Zip, + Unknown, +} diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/Log.cs b/tools/assembly-store-reader-mk2/AssemblyStore/Log.cs new file mode 100644 index 00000000000..497b6c430d6 --- /dev/null +++ b/tools/assembly-store-reader-mk2/AssemblyStore/Log.cs @@ -0,0 +1,12 @@ +using System; + +namespace Xamarin.Android.AssemblyStore; + +static class Log +{ + public static void Debug (string message) + { + // TODO: verbosity + Console.WriteLine (message); + } +} diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V1.cs b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V1.cs new file mode 100644 index 00000000000..d907721c088 --- /dev/null +++ b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V1.cs @@ -0,0 +1,34 @@ +using System.Collections.Generic; +using System.IO; + +namespace Xamarin.Android.AssemblyStore; + +class StoreReader_V1 : AssemblyStoreReader +{ + public override string Description => "Assembly store v1"; + public override bool NeedsExtensionInName => false; + + public static IList ApkPaths { get; } + public static IList AabPaths { get; } + public static IList AabBasePaths { get; } + + static StoreReader_V1 () + { + ApkPaths = new List ().AsReadOnly (); + AabPaths = new List ().AsReadOnly (); + AabBasePaths = new List ().AsReadOnly (); + } + + public StoreReader_V1 (Stream store, string path) + : base (store, path) + {} + + protected override bool IsSupported () + { + return false; + } + + protected override void Prepare () + { + } +} diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.Classes.cs b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.Classes.cs new file mode 100644 index 00000000000..9dd8a19a054 --- /dev/null +++ b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.Classes.cs @@ -0,0 +1,94 @@ +using System.Collections.Generic; + +using Xamarin.Android.Tools; + +namespace Xamarin.Android.AssemblyStore; + +partial class StoreReader_V2 +{ + sealed class Header + { + public const uint NativeSize = 5 * sizeof (uint); + + public readonly uint magic; + public readonly uint version; + public readonly uint entry_count; + public readonly uint index_entry_count; + + // Index size in bytes + public readonly uint index_size; + + public Header (uint magic, uint version, uint entry_count, uint index_entry_count, uint index_size) + { + this.magic = magic; + this.version = version; + this.entry_count = entry_count; + this.index_entry_count = index_entry_count; + this.index_size = index_size; + } + } + + sealed class IndexEntry + { + public readonly ulong name_hash; + public readonly uint descriptor_index; + + public IndexEntry (ulong name_hash, uint descriptor_index) + { + this.name_hash = name_hash; + this.descriptor_index = descriptor_index; + } + } + + sealed class EntryDescriptor + { + public uint mapping_index; + + public uint data_offset; + public uint data_size; + + public uint debug_data_offset; + public uint debug_data_size; + + public uint config_data_offset; + public uint config_data_size; + } + + sealed class StoreItem_V2 : AssemblyStoreItem + { + public StoreItem_V2 (AndroidTargetArch targetArch, string name, bool is64Bit, List indexEntries, EntryDescriptor descriptor) + : base (name, is64Bit, IndexToHashes (indexEntries)) + { + DataOffset = descriptor.data_offset; + DataSize = descriptor.data_size; + DebugOffset = descriptor.debug_data_offset; + DebugSize = descriptor.debug_data_size; + ConfigOffset = descriptor.config_data_offset; + ConfigSize = descriptor.config_data_size; + TargetArch = targetArch; + } + + static List IndexToHashes (List indexEntries) + { + var ret = new List (); + foreach (IndexEntry ie in indexEntries) { + ret.Add (ie.name_hash); + } + + return ret; + } + } + + sealed class TemporaryItem + { + public readonly string Name; + public readonly List IndexEntries = new List (); + public readonly EntryDescriptor Descriptor; + + public TemporaryItem (string name, EntryDescriptor descriptor) + { + Name = name; + Descriptor = descriptor; + } + } +} diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs new file mode 100644 index 00000000000..f434139387a --- /dev/null +++ b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs @@ -0,0 +1,195 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +using Xamarin.Android.Tools; +using Xamarin.Android.Tasks; + +namespace Xamarin.Android.AssemblyStore; + +partial class StoreReader_V2 : AssemblyStoreReader +{ + // Bit 31 is set for 64-bit platforms, cleared for the 32-bit ones + const uint ASSEMBLY_STORE_FORMAT_VERSION_64BIT = 0x80000002; // Must match the ASSEMBLY_STORE_FORMAT_VERSION native constant + const uint ASSEMBLY_STORE_FORMAT_VERSION_32BIT = 0x00000002; + const uint ASSEMBLY_STORE_FORMAT_VERSION_MASK = 0xF0000000; + + const uint ASSEMBLY_STORE_ABI_AARCH64 = 0x00010000; + const uint ASSEMBLY_STORE_ABI_ARM = 0x00020000; + const uint ASSEMBLY_STORE_ABI_X64 = 0x00030000; + const uint ASSEMBLY_STORE_ABI_X86 = 0x00040000; + const uint ASSEMBLY_STORE_ABI_MASK = 0x00FF0000; + + public override string Description => "Assembly store v2"; + public override bool NeedsExtensionInName => true; + + public static IList ApkPaths { get; } + public static IList AabPaths { get; } + public static IList AabBasePaths { get; } + + readonly HashSet supportedVersions; + + Header? header; + + static StoreReader_V2 () + { + var paths = new List { + GetArchPath (AndroidTargetArch.Arm64), + GetArchPath (AndroidTargetArch.Arm), + GetArchPath (AndroidTargetArch.X86_64), + GetArchPath (AndroidTargetArch.X86), + }; + ApkPaths = paths.AsReadOnly (); + AabBasePaths = ApkPaths; + + const string AabBaseDir = "base"; + paths = new List { + GetArchPath (AndroidTargetArch.Arm64, AabBaseDir), + GetArchPath (AndroidTargetArch.Arm, AabBaseDir), + GetArchPath (AndroidTargetArch.X86_64, AabBaseDir), + GetArchPath (AndroidTargetArch.X86, AabBaseDir), + }; + AabPaths = paths.AsReadOnly (); + + string GetArchPath (AndroidTargetArch arch, string? root = null) + { + const string LibDirName = "lib"; + + string abi = MonoAndroidHelper.ArchToAbi (arch); + var parts = new List (); + if (!String.IsNullOrEmpty (root)) { + parts.Add (LibDirName); + } else { + root = LibDirName; + } + parts.Add (abi); + parts.Add (GetBlobName (abi)); + + return MonoAndroidHelper.MakeZipArchivePath (root, parts); + } + } + + public StoreReader_V2 (Stream store, string path) + : base (store, path) + { + supportedVersions = new HashSet { + ASSEMBLY_STORE_FORMAT_VERSION_64BIT | ASSEMBLY_STORE_ABI_AARCH64, + ASSEMBLY_STORE_FORMAT_VERSION_64BIT | ASSEMBLY_STORE_ABI_X64, + ASSEMBLY_STORE_FORMAT_VERSION_32BIT | ASSEMBLY_STORE_ABI_ARM, + ASSEMBLY_STORE_FORMAT_VERSION_32BIT | ASSEMBLY_STORE_ABI_X86, + }; + } + + static string GetBlobName (string abi) => $"libassemblies.{abi}.blob.so"; + + protected override bool IsSupported () + { + StoreStream.Seek (0, SeekOrigin.Begin); + using var reader = CreateReader (); + + uint magic = reader.ReadUInt32 (); + if (magic != Utils.ASSEMBLY_STORE_MAGIC) { + Log.Debug ($"Store '{StorePath}' has invalid header magic number."); + return false; + } + + uint version = reader.ReadUInt32 (); + if (!supportedVersions.Contains (version)) { + Log.Debug ($"Store '{StorePath}' has unsupported version 0x{version:x}"); + return false; + } + + uint entry_count = reader.ReadUInt32 (); + uint index_entry_count = reader.ReadUInt32 (); + uint index_size = reader.ReadUInt32 (); + + header = new Header (magic, version, entry_count, index_entry_count, index_size); + return true; + } + + protected override void Prepare () + { + if (header == null) { + throw new InvalidOperationException ("Internal error: header not set, was IsSupported() called?"); + } + + TargetArch = (header.version & ASSEMBLY_STORE_ABI_MASK) switch { + ASSEMBLY_STORE_ABI_AARCH64 => AndroidTargetArch.Arm64, + ASSEMBLY_STORE_ABI_ARM => AndroidTargetArch.Arm, + ASSEMBLY_STORE_ABI_X64 => AndroidTargetArch.X86_64, + ASSEMBLY_STORE_ABI_X86 => AndroidTargetArch.X86, + _ => throw new NotSupportedException ($"Unsupported ABI in store version: 0x{header.version:x}") + }; + + Is64Bit = (header.version & ASSEMBLY_STORE_FORMAT_VERSION_MASK) != 0; + AssemblyCount = header.entry_count; + IndexEntryCount = header.index_entry_count; + + StoreStream.Seek (Header.NativeSize, SeekOrigin.Begin); + using var reader = CreateReader (); + + var index = new List (); + for (uint i = 0; i < header.index_entry_count; i++) { + ulong name_hash; + if (Is64Bit) { + name_hash = reader.ReadUInt64 (); + } else { + name_hash = (ulong)reader.ReadUInt32 (); + } + + uint descriptor_index = reader.ReadUInt32 (); + index.Add (new IndexEntry (name_hash, descriptor_index)); + } + + var descriptors = new List (); + for (uint i = 0; i < header.entry_count; i++) { + uint mapping_index = reader.ReadUInt32 (); + uint data_offset = reader.ReadUInt32 (); + uint data_size = reader.ReadUInt32 (); + uint debug_data_offset = reader.ReadUInt32 (); + uint debug_data_size = reader.ReadUInt32 (); + uint config_data_offset = reader.ReadUInt32 (); + uint config_data_size = reader.ReadUInt32 (); + + var desc = new EntryDescriptor { + mapping_index = mapping_index, + data_offset = data_offset, + data_size = data_size, + debug_data_offset = debug_data_offset, + debug_data_size = debug_data_size, + config_data_offset = config_data_offset, + config_data_size = config_data_size, + }; + descriptors.Add (desc); + } + + var names = new List (); + for (uint i = 0; i < header.entry_count; i++) { + uint name_length = reader.ReadUInt32 (); + byte[] name_bytes = reader.ReadBytes ((int)name_length); + names.Add (Encoding.UTF8.GetString (name_bytes)); + } + + var tempItems = new Dictionary (); + foreach (IndexEntry ie in index) { + if (!tempItems.TryGetValue (ie.descriptor_index, out TemporaryItem? item)) { + item = new TemporaryItem (names[(int)ie.descriptor_index], descriptors[(int)ie.descriptor_index]); + tempItems.Add (ie.descriptor_index, item); + } + item.IndexEntries.Add (ie); + } + + if (tempItems.Count != descriptors.Count) { + throw new InvalidOperationException ($"Assembly store '{StorePath}' index is corrupted."); + } + + var storeItems = new List (); + foreach (var kvp in tempItems) { + TemporaryItem ti = kvp.Value; + var item = new StoreItem_V2 (TargetArch, ti.Name, Is64Bit, ti.IndexEntries, ti.Descriptor); + storeItems.Add (item); + } + Assemblies = storeItems.AsReadOnly (); + } +} diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/Utils.cs b/tools/assembly-store-reader-mk2/AssemblyStore/Utils.cs new file mode 100644 index 00000000000..d36d86fbd71 --- /dev/null +++ b/tools/assembly-store-reader-mk2/AssemblyStore/Utils.cs @@ -0,0 +1,85 @@ +using System; +using System.IO; +using System.Buffers; + +using Xamarin.Tools.Zip; + +namespace Xamarin.Android.AssemblyStore; + +static class Utils +{ + static readonly string[] aabZipEntries = { + "base/manifest/AndroidManifest.xml", + "BundleConfig.pb", + }; + + static readonly string[] aabBaseZipEntries = { + "manifest/AndroidManifest.xml", + }; + + static readonly string[] apkZipEntries = { + "AndroidManifest.xml", + }; + + public const uint ZIP_MAGIC = 0x4034b50; + public const uint ASSEMBLY_STORE_MAGIC = 0x41424158; + + public static readonly ArrayPool BytePool = ArrayPool.Shared; + + public static (FileFormat format, FileInfo? info) DetectFileFormat (string path) + { + if (String.IsNullOrEmpty (path)) { + return (FileFormat.Unknown, null); + } + + var info = new FileInfo (path); + if (!info.Exists) { + return (FileFormat.Unknown, null); + } + + using var reader = new BinaryReader (info.OpenRead ()); + + // ATM, all formats we recognize have 4-byte magic at the start + FileFormat format = reader.ReadUInt32 () switch { + Utils.ZIP_MAGIC => FileFormat.Zip, + Utils.ASSEMBLY_STORE_MAGIC => FileFormat.AssemblyStore, + _ => FileFormat.Unknown + }; + + if (format == FileFormat.Unknown || format != FileFormat.Zip) { + return (format, info); + } + + return (DetectAndroidArchive (info, format), info); + } + + static FileFormat DetectAndroidArchive (FileInfo info, FileFormat defaultFormat) + { + using var zip = ZipArchive.Open (info.FullName, FileMode.Open); + + if (HasAllEntries (zip, aabZipEntries)) { + return FileFormat.Aab; + } + + if (HasAllEntries (zip, apkZipEntries)) { + return FileFormat.Apk; + } + + if (HasAllEntries (zip, aabBaseZipEntries)) { + return FileFormat.AabBase; + } + + return defaultFormat; + } + + static bool HasAllEntries (ZipArchive zip, string[] entries) + { + foreach (string entry in entries) { + if (!zip.ContainsEntry (entry, caseSensitive: true)) { + return false; + } + } + + return true; + } +} diff --git a/tools/assembly-store-reader-mk2/Directory.Build.targets b/tools/assembly-store-reader-mk2/Directory.Build.targets new file mode 100644 index 00000000000..e58eed5ca2c --- /dev/null +++ b/tools/assembly-store-reader-mk2/Directory.Build.targets @@ -0,0 +1,6 @@ + + + + + diff --git a/tools/assembly-store-reader-mk2/Main.cs b/tools/assembly-store-reader-mk2/Main.cs new file mode 100644 index 00000000000..24767dfae88 --- /dev/null +++ b/tools/assembly-store-reader-mk2/Main.cs @@ -0,0 +1,118 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +using Mono.Options; +using Xamarin.Android.Tasks; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.AssemblyStore; + +class App +{ + static void ShowHelp () + { + } + + static int WriteErrorAndReturn (string message) + { + Console.Error.WriteLine (message); + return 1; + } + + static HashSet? ParseArchList (string values) + { + if (String.IsNullOrEmpty (values)) { + return null; + } + + var ret = new HashSet (); + foreach (string a in values.Split (',')) { + string archName = a.Trim (); + if (Enum.TryParse (archName, out AndroidTargetArch arch)) { + ret.Add (arch); + continue; + } + + arch = archName.ToLowerInvariant () switch { + "aarch64" => AndroidTargetArch.Arm64, + "arm32" => AndroidTargetArch.Arm, + "arm64" => AndroidTargetArch.Arm64, + "armv7a" => AndroidTargetArch.Arm, + "armv8a" => AndroidTargetArch.Arm64, + "x64" => AndroidTargetArch.X86_64, + _ => throw new InvalidOperationException ($"Unknown architecture name '{archName}'") + }; + ret.Add (arch); + } + + return ret; + } + + static string GetArchNames () + { + return String.Join (", ", MonoAndroidHelper.SupportedTargetArchitectures.Select (a => a.ToString ().ToLowerInvariant ())); + } + + static int Main (string[] args) + { + HashSet? arches = null; + bool showHelp = false; + + var options = new OptionSet { + "Usage: read-assembly-store [OPTIONS] BLOB_PATH", + "", + " where each BLOB_PATH can point to:", + " * aab file", + " * apk file", + " * index store file (e.g. base_assemblies.blob or assemblies.arm64_v8a.blob.so)", + " * arch store file (e.g. base_assemblies.arm64_v8a.blob)", + " * store manifest file (e.g. base_assemblies.manifest)", + " * store base name (e.g. base or base_assemblies)", + "", + " In each case the whole set of stores and manifests will be read (if available). Search for the", + " various members of the store set (common/main store, arch stores, manifest) is based on this naming", + " convention:", + "", + " {BASE_NAME}[.ARCH_NAME].{blob|blob.so|manifest}", + "", + " Whichever file is referenced in `BLOB_PATH`, the BASE_NAME component is extracted and all the found files are read.", + " If `BLOB_PATH` points to an aab or an apk, BASE_NAME will always be `assemblies`", + "", + {"a|arch=", $"Limit listing of assemblies to these {{ARCHITECTURES}} only. A comma-separated list of one or more of: {GetArchNames ()}", v => arches = ParseArchList (v) }, + "", + {"?|h|help", "Show this help screen", v => showHelp = true}, + }; + + List? theRest = options.Parse (args); + if (theRest == null || theRest.Count == 0 || showHelp) { + options.WriteOptionDescriptions (Console.Out); + return showHelp ? 0 : 1; + } + + string inputFile = theRest[0]; + (FileFormat format, FileInfo? info) = Utils.DetectFileFormat (inputFile); + if (info == null) { + return WriteErrorAndReturn ($"File '{inputFile}' does not exist."); + } + + (IList? explorers, string? errorMessage) = AssemblyStoreExplorer.Open (inputFile); + if (explorers == null) { + return WriteErrorAndReturn (errorMessage ?? "Unknown error"); + } + + foreach (AssemblyStoreExplorer store in explorers) { + if (arches != null && store.TargetArch.HasValue && !arches.Contains (store.TargetArch.Value)) { + continue; + } + + var printer = new StorePrettyPrinter (store); + printer.Show (); + } + + return 0; + } + + +} diff --git a/tools/assembly-store-reader-mk2/StorePrettyPrinter.cs b/tools/assembly-store-reader-mk2/StorePrettyPrinter.cs new file mode 100644 index 00000000000..8efd16fd162 --- /dev/null +++ b/tools/assembly-store-reader-mk2/StorePrettyPrinter.cs @@ -0,0 +1,109 @@ +using System; +using System.Collections.Generic; +using System.Text; + +using Xamarin.Android.Tools; + +namespace Xamarin.Android.AssemblyStore; + +class StorePrettyPrinter +{ + AssemblyStoreExplorer explorer; + + public StorePrettyPrinter (AssemblyStoreExplorer storeExplorer) + { + explorer = storeExplorer; + } + + public void Show () + { + Console.WriteLine ($"Store: {explorer.StorePath}"); + Console.WriteLine ($" Target architecture: {GetTargetArch (explorer)} ({GetBitness (explorer.Is64Bit)}-bit)"); + Console.WriteLine ($" Assembly count: {explorer.AssemblyCount}"); + Console.WriteLine ($" Index entry count: {explorer.IndexEntryCount}"); + Console.WriteLine (); + + if (explorer.Assemblies == null || explorer.Assemblies.Count == 0) { + Console.WriteLine ("NO ASSEMBLIES!"); + return; + } + + var assemblies = new List (explorer.Assemblies); + assemblies.Sort ((AssemblyStoreItem a, AssemblyStoreItem b) => a.Name.CompareTo (b.Name)); + + Console.WriteLine ("Assemblies:"); + var line = new StringBuilder (); + foreach (AssemblyStoreItem assembly in assemblies) { + line.Clear (); + line.Append (" "); + line.AppendLine (assembly.Name); + line.Append (" PE image data: "); + FormatOffsetAndSize (line, assembly.DataOffset, assembly.DataSize); + line.AppendLine (); + line.Append (" Debug data: "); + FormatOffsetAndSize (line, assembly.DebugOffset, assembly.DebugSize); + line.AppendLine (); + line.Append (" Config data: "); + FormatOffsetAndSize (line, assembly.ConfigOffset, assembly.ConfigSize); + line.AppendLine (); + line.Append (" Name hashes: "); + FormatHashes (line, assembly.Hashes); + line.AppendLine (); + Console.WriteLine (line.ToString ()); + } + Console.WriteLine (); + } + + static void FormatOffsetAndSize (StringBuilder sb, uint offset, uint size) + { + if (offset == 0) { + FormatNone (sb); + return; + } + + sb.Append ("offset "); + sb.Append (offset); + sb.Append (", size "); + sb.Append (size); + } + + static void FormatHashes (StringBuilder sb, IList hashes) + { + if (hashes.Count == 0) { + FormatNone (sb); + return; + } + + bool first = true; + foreach (ulong hash in hashes) { + if (first) { + first = false; + } else { + sb.Append (", "); + } + sb.Append ($"0x{hash:x}"); + } + } + + static void FormatNone (StringBuilder sb) + { + sb.Append ("none"); + } + + static string GetBitness (bool is64bit) => is64bit ? "64" : "32"; + + static string GetTargetArch (AssemblyStoreExplorer storeExplorer) + { + if (storeExplorer.TargetArch == null) { + return "ABI agnostic"; + } + + return storeExplorer.TargetArch switch { + AndroidTargetArch.Arm64 => "Arm64", + AndroidTargetArch.Arm => "Arm32", + AndroidTargetArch.X86_64 => "x64", + AndroidTargetArch.X86 => "x86", + _ => throw new NotSupportedException ($"Unsupported target architecture {storeExplorer.TargetArch}") + }; + } +} diff --git a/tools/assembly-store-reader-mk2/assembly-store-reader.csproj b/tools/assembly-store-reader-mk2/assembly-store-reader.csproj new file mode 100644 index 00000000000..81520d99671 --- /dev/null +++ b/tools/assembly-store-reader-mk2/assembly-store-reader.csproj @@ -0,0 +1,33 @@ + + + + + Microsoft Corporation + 2023 Microsoft Corporation + 0.0.2 + false + ../../bin/$(Configuration)/bin/assembly-store-reader + Exe + $(DotNetStableTargetFramework) + Xamarin.Android.AssemblyStoreReader + disable + enable + + + + + + + + + + + + + + + + + + + diff --git a/tools/assembly-store-reader/Program.cs b/tools/assembly-store-reader/Program.cs index 6052ed396ce..84dbe5703c6 100644 --- a/tools/assembly-store-reader/Program.cs +++ b/tools/assembly-store-reader/Program.cs @@ -123,7 +123,7 @@ static int Main (string[] args) Console.Error.WriteLine (@" where each BLOB_PATH can point to: * aab file * apk file - * index store file (e.g. base_assemblies.blob) + * index store file (e.g. base_assemblies.blob or assemblies.arm64_v8a.blob.so) * arch store file (e.g. base_assemblies.arm64_v8a.blob) * store manifest file (e.g. base_assemblies.manifest) * store base name (e.g. base or base_assemblies) diff --git a/tools/assembly-store-reader/assembly-store-reader.csproj b/tools/assembly-store-reader/assembly-store-reader.csproj index 4e85b5fc9d5..9b0e604b1da 100644 --- a/tools/assembly-store-reader/assembly-store-reader.csproj +++ b/tools/assembly-store-reader/assembly-store-reader.csproj @@ -3,8 +3,8 @@ Microsoft Corporation - 2021 Microsoft Corporation - 0.0.1 + 2023 Microsoft Corporation + 0.0.2 false ../../bin/$(Configuration)/bin/assembly-store-reader Exe diff --git a/tools/decompress-assemblies/decompress-assemblies.csproj b/tools/decompress-assemblies/decompress-assemblies.csproj index d073ae2322d..1f789958973 100644 --- a/tools/decompress-assemblies/decompress-assemblies.csproj +++ b/tools/decompress-assemblies/decompress-assemblies.csproj @@ -21,8 +21,6 @@ - -