Skip to content

Commit

Permalink
[Xamarin.Android.Build.Tasks] use System.Reflection.Metadata in <Reso…
Browse files Browse the repository at this point in the history
…lveAssemblies/>

Context: https://github.com/dotnet/corefx/tree/master/src/System.Reflection.Metadata/src/System/Reflection/Metadata
Context: https://github.com/jonathanpeppers/Benchmarks

There is a new System.Reflection.Metadata library from corefx for
reading .NET assemblies. It is a bit more performant than Mono.Cecil
because it is a different library with different opinions.

Some notes about System.Reflection.Metadata:

- SRM has a forward "reader" style API
- SRM uses lots of structs, and you have to do an additional call to
lookup strings generally.
- SRM, as far as I have seen, doesn't have APIs to modify and write
out new assemblies.
- SRM only supports "portable" pdb files.
- SRM is not well documented yet. To discover usage, I read source
  code and/or unit tests.

From my benchmark above, it seems that SRM is 10x faster on
Windows/.NET framework and 5x faster on macOS/Mono.

So it makes sense for use to use SRM when reading assemblies (and we
don't need symbols), and continue with Mono.Cecil for the linker and
other things that modify assemblies.

There are a few places we can take advantage of SRM, but the simplest
with a reasonable impact was `ResolveAssemblies`:

    Before:
    320 ms  ResolveAssemblies                          1 calls
    After:
    112 ms  ResolveAssemblies                          1 calls

So a ~200ms savings on this MSBuild task, which runs on *every* build.
This was the Xamarin.Forms test project in this repo: a build with no
changes.

~~ Changes ~~

- Added a `MetadataResolver` type, as a way to cache `PEReader`
  instances. This is a comparable drop-in replacement for
  `DirectoryAssemblyResolver`.
- `MonoAndroidHelper.IsReferenceAssembly` now uses
  `System.Reflection.Metadata` instead of `Mono.Cecil`. This is used
  in a few other MSBuild tasks.
- A `MetadataExtensions` provides an extension method to simplify
  getting the full name of a custom attribute. We can add more here as
  needed.
- Had to adjust the filename reported for XA2002, should optionally
  call `Path.GetFileNameWithoutExtension` if the name ends with
  `.dll`.

The resulting code *should* be the same, except we are using SRM over
Mono.Cecil.

~~ Other changes ~~

[xabuild.exe] remove SRM reference

This appears to fix the build on macOS, we had this workaround from a
mono bump in the past. Since xabuild has its own version of
System.Reflection.Metadata that was already loaded, we weren't loading
the one we are using in XA's MSBuild tasks.

Things appear to work without the reference now.

~~ Downstream ~~

We will need to add the following assemblies to the installer:

- `System.Reflection.Metadata.dll`
- `System.Collections.Immutable.dll`
  • Loading branch information
jonathanpeppers committed Jan 14, 2019
1 parent 434d2d8 commit 1da8240
Show file tree
Hide file tree
Showing 8 changed files with 199 additions and 83 deletions.
119 changes: 60 additions & 59 deletions src/Xamarin.Android.Build.Tasks/Tasks/ResolveAssemblies.cs
Original file line number Diff line number Diff line change
@@ -1,21 +1,18 @@
// Copyright (C) 2011, Xamarin Inc.
// Copyright (C) 2010, Novell Inc.

using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;
using Mono.Cecil;
using MonoDroid.Tuner;
using NuGet.Frameworks;
using NuGet.ProjectModel;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection.Metadata;
using System.Text;
using Xamarin.Android.Tools;
using NuGet.Common;
using NuGet.Frameworks;
using NuGet.ProjectModel;

using Java.Interop.Tools.Cecil;

namespace Xamarin.Android.Tasks
{
Expand Down Expand Up @@ -62,7 +59,7 @@ public override bool Execute ()
Yield ();
try {
System.Threading.Tasks.Task.Run (() => {
using (var resolver = new DirectoryAssemblyResolver (this.CreateTaskLogger (), loadDebugSymbols: false)) {
using (var resolver = new MetadataResolver ()) {
Execute (resolver);
}
}, Token).ContinueWith (Complete);
Expand All @@ -72,14 +69,13 @@ public override bool Execute ()
}
}

void Execute (DirectoryAssemblyResolver resolver)
void Execute (MetadataResolver resolver)
{
foreach (var dir in ReferenceAssembliesDirectory.Split (new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries))
resolver.SearchDirectories.Add (dir);

var assemblies = new Dictionary<string, ITaskItem> ();
foreach (var dir in ReferenceAssembliesDirectory.Split (new char [] { ';' }, StringSplitOptions.RemoveEmptyEntries))
resolver.AddSearchDirectory (dir);

var topAssemblyReferences = new List<AssemblyDefinition> ();
var assemblies = new Dictionary<string, ITaskItem> (Assemblies.Length);
var topAssemblyReferences = new List<string> (Assemblies.Length);
var logger = new NuGetLogger((s) => {
LogDebugMessage ("{0}", s);
});
Expand All @@ -92,32 +88,28 @@ void Execute (DirectoryAssemblyResolver resolver)
try {
foreach (var assembly in Assemblies) {
var assembly_path = Path.GetDirectoryName (assembly.ItemSpec);

if (!resolver.SearchDirectories.Contains (assembly_path))
resolver.SearchDirectories.Add (assembly_path);
resolver.AddSearchDirectory (assembly_path);

// Add each user assembly and all referenced assemblies (recursive)
var assemblyDef = resolver.Load (assembly.ItemSpec);
if (assemblyDef == null)
throw new InvalidOperationException ("Failed to load assembly " + assembly.ItemSpec);
if (MonoAndroidHelper.IsReferenceAssembly (assemblyDef)) {
string resolved_assembly = resolver.Resolve (assembly.ItemSpec);
if (MonoAndroidHelper.IsReferenceAssembly (resolved_assembly)) {
// Resolve "runtime" library
var asmFullPath = Path.GetFullPath (assembly.ItemSpec);
if (lockFile != null)
assemblyDef = ResolveRuntimeAssemblyForReferenceAssembly (lockFile, resolver, asmFullPath);
if (lockFile == null || assemblyDef == null) {
LogCodedWarning ("XA0107", asmFullPath, 0, "Ignoring {0} as it is a Reference Assembly", asmFullPath);
resolved_assembly = ResolveRuntimeAssemblyForReferenceAssembly (lockFile, assembly.ItemSpec);
if (lockFile == null || resolved_assembly == null) {
LogCodedWarning ("XA0107", resolved_assembly, 0, "Ignoring {0} as it is a Reference Assembly", resolved_assembly);
continue;
}
}
topAssemblyReferences.Add (assemblyDef);
topAssemblyReferences.Add (resolved_assembly);
var taskItem = new TaskItem (assembly) {
ItemSpec = Path.GetFullPath (assemblyDef.MainModule.FileName),
ItemSpec = Path.GetFullPath (resolved_assembly),
};
if (string.IsNullOrEmpty (taskItem.GetMetadata ("ReferenceAssembly"))) {
taskItem.SetMetadata ("ReferenceAssembly", taskItem.ItemSpec);
}
assemblies [assemblyDef.Name.Name] = taskItem;
string assemblyName = Path.GetFileNameWithoutExtension (resolved_assembly);
assemblies [assemblyName] = taskItem;
}
} catch (Exception ex) {
LogError ("Exception while loading assemblies: {0}", ex);
Expand Down Expand Up @@ -171,7 +163,7 @@ void Execute (DirectoryAssemblyResolver resolver)
readonly Dictionary<string, int> api_levels = new Dictionary<string, int> ();
int indent = 2;

AssemblyDefinition ResolveRuntimeAssemblyForReferenceAssembly (LockFile lockFile, DirectoryAssemblyResolver resolver, string assemblyPath)
string ResolveRuntimeAssemblyForReferenceAssembly (LockFile lockFile, string assemblyPath)
{
if (string.IsNullOrEmpty(TargetMoniker))
return null;
Expand Down Expand Up @@ -200,16 +192,16 @@ AssemblyDefinition ResolveRuntimeAssemblyForReferenceAssembly (LockFile lockFile
path = Path.Combine (folder.Path, libraryPath.Path, runtime.Path).Replace('/', Path.DirectorySeparatorChar);
if (!File.Exists (path))
continue;
LogDebugMessage ($"Attempting to load {path}");
return resolver.Load (path, forceLoad: true);
return path;
}
return null;
}

void AddAssemblyReferences (DirectoryAssemblyResolver resolver, Dictionary<string, ITaskItem> assemblies, AssemblyDefinition assembly, List<string> resolutionPath)
void AddAssemblyReferences (MetadataResolver resolver, Dictionary<string, ITaskItem> assemblies, string assemblyPath, List<string> resolutionPath)
{
var assemblyName = assembly.Name.Name;
var fullPath = Path.GetFullPath (assembly.MainModule.FileName);
var reader = resolver.GetAssemblyReader (assemblyPath);
var assembly = reader.GetAssemblyDefinition ();
var assemblyName = reader.GetString (assembly.Name);

// Don't repeat assemblies we've already done
bool topLevel = resolutionPath == null;
Expand All @@ -219,22 +211,23 @@ void AddAssemblyReferences (DirectoryAssemblyResolver resolver, Dictionary<strin
if (resolutionPath == null)
resolutionPath = new List<string>();

CheckAssemblyAttributes (assembly);
CheckAssemblyAttributes (assembly, reader);

LogMessage ("{0}Adding assembly reference for {1}, recursively...", new string (' ', indent), assembly.Name);
resolutionPath.Add (assembly.Name.Name);
LogMessage ("{0}Adding assembly reference for {1}, recursively...", new string (' ', indent), assemblyName);
resolutionPath.Add (assemblyName);
indent += 2;

// Add this assembly
if (!topLevel) {
assemblies [assemblyName] = CreateAssemblyTaskItem (fullPath);
assemblies [assemblyName] = CreateAssemblyTaskItem (Path.GetFullPath (assemblyPath));
}

// Recurse into each referenced assembly
foreach (AssemblyNameReference reference in assembly.MainModule.AssemblyReferences) {
AssemblyDefinition reference_assembly;
foreach (var handle in reader.AssemblyReferences) {
var reference = reader.GetAssemblyReference (handle);
string reference_assembly;
try {
reference_assembly = resolver.Resolve (reference);
reference_assembly = resolver.Resolve (reader.GetString (reference.Name));
} catch (FileNotFoundException ex) {
var references = new StringBuilder ();
for (int i = 0; i < resolutionPath.Count; i++) {
Expand All @@ -245,7 +238,10 @@ void AddAssemblyReferences (DirectoryAssemblyResolver resolver, Dictionary<strin
references.Append ('`');
}

string missingAssembly = Path.GetFileNameWithoutExtension (ex.FileName);
string missingAssembly = ex.FileName;
if (missingAssembly.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) {
missingAssembly = Path.GetFileNameWithoutExtension (missingAssembly);
}
string message = $"Can not resolve reference: `{missingAssembly}`, referenced by {references}.";
if (MonoAndroidHelper.IsFrameworkAssembly (ex.FileName)) {
LogCodedError ("XA2002", $"{message} Perhaps it doesn't exist in the Mono for Android profile?");
Expand All @@ -261,25 +257,30 @@ void AddAssemblyReferences (DirectoryAssemblyResolver resolver, Dictionary<strin
resolutionPath.RemoveAt (resolutionPath.Count - 1);
}

void CheckAssemblyAttributes (AssemblyDefinition assembly)
void CheckAssemblyAttributes (AssemblyDefinition assembly, MetadataReader reader)
{
foreach (var att in assembly.CustomAttributes) {
switch (att.AttributeType.FullName) {
foreach (var handle in assembly.GetCustomAttributes ()) {
var attribute = reader.GetCustomAttribute (handle);
switch (reader.GetCustomAttributeFullName (attribute)) {
case "Java.Interop.DoNotPackageAttribute": {
string file = (string)att.ConstructorArguments.First ().Value;
if (string.IsNullOrWhiteSpace (file))
LogError ("In referenced assembly {0}, Java.Interop.DoNotPackageAttribute requires non-null file name.", assembly.FullName);
do_not_package_atts.Add (Path.GetFileName (file));
var decoded = attribute.DecodeValue (DummyCustomAttributeProvider.Instance);
if (decoded.FixedArguments.Length > 0) {
string file = decoded.FixedArguments [0].Value?.ToString ();
if (string.IsNullOrWhiteSpace (file))
LogError ("In referenced assembly {0}, Java.Interop.DoNotPackageAttribute requires non-null file name.", assembly.GetAssemblyName ().FullName);
do_not_package_atts.Add (Path.GetFileName (file));
}
}
break;
case "System.Runtime.Versioning.TargetFrameworkAttribute": {
foreach (var p in att.ConstructorArguments) {
var value = p.Value.ToString ();
if (value.StartsWith ("MonoAndroid")) {
var decoded = attribute.DecodeValue (DummyCustomAttributeProvider.Instance);
foreach (var p in decoded.FixedArguments) {
var value = p.Value?.ToString ();
if (value != null && value.StartsWith ("MonoAndroid", StringComparison.Ordinal)) {
var values = value.Split ('=');
var apiLevel = MonoAndroidHelper.SupportedVersions.GetApiLevelFromFrameworkVersion (values [1]);
if (apiLevel != null) {
var assemblyName = assembly.Name.Name;
var assemblyName = reader.GetString (assembly.Name);
Log.LogDebugMessage ("{0}={1}", assemblyName, apiLevel);
api_levels [assemblyName] = apiLevel.Value;
}
Expand All @@ -305,7 +306,7 @@ static LinkModes ParseLinkMode (string linkmode)
return mode;
}

void AddI18nAssemblies (DirectoryAssemblyResolver resolver, Dictionary<string, ITaskItem> assemblies)
void AddI18nAssemblies (MetadataResolver resolver, Dictionary<string, ITaskItem> assemblies)
{
var i18n = Linker.ParseI18nAssemblies (I18nAssemblies);
var link = ParseLinkMode (LinkMode);
Expand All @@ -332,10 +333,10 @@ void AddI18nAssemblies (DirectoryAssemblyResolver resolver, Dictionary<string, I
ResolveI18nAssembly (resolver, "I18N.West", assemblies);
}

void ResolveI18nAssembly (DirectoryAssemblyResolver resolver, string name, Dictionary<string, ITaskItem> assemblies)
void ResolveI18nAssembly (MetadataResolver resolver, string name, Dictionary<string, ITaskItem> assemblies)
{
var assembly = resolver.Resolve (AssemblyNameReference.Parse (name));
var assemblyFullPath = Path.GetFullPath (assembly.MainModule.FileName);
var assembly = resolver.Resolve (name);
var assemblyFullPath = Path.GetFullPath (assembly);
assemblies [name] = CreateAssemblyTaskItem (assemblyFullPath);
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
using System.Reflection.Metadata;

namespace Xamarin.Android.Tasks
{
/// <summary>
/// A helper type for System.Reflection.Metadata. Getting the value of custom attribute arguments is a bit convoluted, if you merely want the values.
///
/// This interface allows usage such as:
/// CustomAttribute attribute = reader.GetCustomAttribute (handle);
/// CustomAttributeValue<object> decoded = attribute.DecodeValue (DummyCustomAttributeProvider.Instance);
/// </summary>
public class DummyCustomAttributeProvider : ICustomAttributeTypeProvider<object>
{
public static readonly DummyCustomAttributeProvider Instance = new DummyCustomAttributeProvider ();

public object GetPrimitiveType (PrimitiveTypeCode typeCode) => null;

public object GetSystemType () => null;

public object GetSZArrayType (object elementType) => null;

public object GetTypeFromDefinition (MetadataReader reader, TypeDefinitionHandle handle, byte rawTypeKind) => null;

public object GetTypeFromReference (MetadataReader reader, TypeReferenceHandle handle, byte rawTypeKind) => null;

public object GetTypeFromSerializedName (string name) => null;

public PrimitiveTypeCode GetUnderlyingEnumType (object type) => default (PrimitiveTypeCode);

public bool IsSystemType (object type) => false;
}
}
21 changes: 21 additions & 0 deletions src/Xamarin.Android.Build.Tasks/Utilities/MetadataExtensions.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
using System.Reflection.Metadata;

namespace Xamarin.Android.Tasks
{
public static class MetadataExtensions
{
public static string GetCustomAttributeFullName (this MetadataReader reader, CustomAttribute attribute)
{
if (attribute.Constructor.Kind == HandleKind.MemberReference) {
var ctor = reader.GetMemberReference ((MemberReferenceHandle)attribute.Constructor);
var type = reader.GetTypeReference ((TypeReferenceHandle)ctor.Parent);
return reader.GetString (type.Namespace) + "." + reader.GetString (type.Name);
} else if (attribute.Constructor.Kind == HandleKind.MethodDefinition) {
var ctor = reader.GetMethodDefinition ((MethodDefinitionHandle)attribute.Constructor);
var type = reader.GetTypeDefinition (ctor.GetDeclaringType ());
return reader.GetString (type.Namespace) + "." + reader.GetString (type.Name);
}
return null;
}
}
}
60 changes: 60 additions & 0 deletions src/Xamarin.Android.Build.Tasks/Utilities/MetadataResolver.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Reflection.Metadata;
using System.Reflection.PortableExecutable;

namespace Xamarin.Android.Tasks
{
/// <summary>
/// A replacement for DirectoryAssemblyResolver, using System.Reflection.Metadata
/// </summary>
public class MetadataResolver : IDisposable
{
readonly Dictionary<string, PEReader> cache = new Dictionary<string, PEReader> ();
readonly List<string> searchDirectories = new List<string> ();

public MetadataReader GetAssemblyReader (string assemblyName)
{
var key = Path.GetFileNameWithoutExtension (assemblyName);
if (!cache.TryGetValue (key, out PEReader reader)) {
var assemblyPath = Resolve (assemblyName);
cache.Add (key, reader = new PEReader (File.OpenRead (assemblyPath)));
}
return reader.GetMetadataReader ();
}

public void AddSearchDirectory (string directory)
{
directory = Path.GetFullPath (directory);
if (!searchDirectories.Contains (directory))
searchDirectories.Add (directory);
}

public string Resolve (string assemblyName)
{
string assemblyPath = assemblyName;
if (!assemblyPath.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) {
assemblyPath += ".dll";
}
if (File.Exists (assemblyPath)) {
return assemblyPath;
}
foreach (var dir in searchDirectories) {
var path = Path.Combine (dir, assemblyPath);
if (File.Exists (path))
return path;
}

throw new FileNotFoundException ($"Could not load assembly '{assemblyName}'.", assemblyName);
}

public void Dispose ()
{
foreach (var provider in cache.Values) {
provider.Dispose ();
}
cache.Clear ();
}
}
}
Loading

0 comments on commit 1da8240

Please sign in to comment.