Skip to content

Commit

Permalink
[Xamarin.Android.Build.Tasks] use System.Reflection.Metadata in <Reso…
Browse files Browse the repository at this point in the history
…lveAssemblies/>

Context: https://github.com/dotnet/corefx/tree/master/src/System.Reflection.Metadata/src/System/Reflection/Metadata
Context: https://github.com/jonathanpeppers/Benchmarks

There is a new System.Reflection.Metadata library from corefx for
reading .NET assemblies. It is a bit more performant than Mono.Cecil
because it is a different library with different opinions.

Some notes about System.Reflection.Metadata:

- SRM has a forward "reader" style API
- SRM uses lots of structs, and you have to do an additional call to
lookup strings generally.
- SRM, as far as I have seen, doesn't have APIs to modify and write
out new assemblies.
- SRM only supports "portable" pdb files.
- SRM is not well documented yet. To discover usage, I read source
  code and/or unit tests.

From my benchmark above, it seems that SRM is 10x faster on
Windows/.NET framework and 5x faster on macOS/Mono.

So it makes sense for use to use SRM when reading assemblies (and we
don't need symbols), and continue with Mono.Cecil for the linker and
other things that modify assemblies.

There are a few places we can take advantage of SRM, but the simplest
with a reasonable impact was `ResolveAssemblies`:

    Before:
    320 ms  ResolveAssemblies                          1 calls
    After:
    112 ms  ResolveAssemblies                          1 calls

So a ~200ms savings on this MSBuild task, which runs on *every* build.
This was the Xamarin.Forms test project in this repo: a build with no
changes.

~~ Changes ~~

- Added a `MetadataResolver` type, as a way to cache `PEReader`
  instances. This is a comparable drop-in replacement for
  `DirectoryAssemblyResolver`.
- `MonoAndroidHelper.IsReferenceAssembly` now uses
  `System.Reflection.Metadata` instead of `Mono.Cecil`. This is used
  in a few other MSBuild tasks.
- A `MetadataExtensions` provides an extension method to simplify
  getting the full name of a custom attribute. We can add more here as
  needed.

The resulting code *should* be the same, except we are using SRM over
Mono.Cecil.

~~ Downstream ~~

We will need to add the following assemblies to the installer:

- `System.Reflection.Metadata.dll`
- `System.Collections.Immutable.dll`
  • Loading branch information
jonathanpeppers committed Jan 11, 2019
1 parent 3576678 commit 2dfcac4
Show file tree
Hide file tree
Showing 7 changed files with 194 additions and 80 deletions.
113 changes: 54 additions & 59 deletions src/Xamarin.Android.Build.Tasks/Tasks/ResolveAssemblies.cs
Original file line number Diff line number Diff line change
@@ -1,21 +1,18 @@
// Copyright (C) 2011, Xamarin Inc.
// Copyright (C) 2010, Novell Inc.

using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;
using Mono.Cecil;
using MonoDroid.Tuner;
using NuGet.Frameworks;
using NuGet.ProjectModel;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection.Metadata;
using System.Text;
using Xamarin.Android.Tools;
using NuGet.Common;
using NuGet.Frameworks;
using NuGet.ProjectModel;

using Java.Interop.Tools.Cecil;

namespace Xamarin.Android.Tasks
{
Expand Down Expand Up @@ -62,7 +59,7 @@ public override bool Execute ()
Yield ();
try {
System.Threading.Tasks.Task.Run (() => {
using (var resolver = new DirectoryAssemblyResolver (this.CreateTaskLogger (), loadDebugSymbols: false)) {
using (var resolver = new MetadataResolver ()) {
Execute (resolver);
}
}, Token).ContinueWith (Complete);
Expand All @@ -72,14 +69,13 @@ public override bool Execute ()
}
}

void Execute (DirectoryAssemblyResolver resolver)
void Execute (MetadataResolver resolver)
{
foreach (var dir in ReferenceAssembliesDirectory.Split (new char[] { ';' }, StringSplitOptions.RemoveEmptyEntries))
resolver.SearchDirectories.Add (dir);

var assemblies = new Dictionary<string, ITaskItem> ();
var assemblies = new Dictionary<string, ITaskItem> (Assemblies.Length);
foreach (var dir in ReferenceAssembliesDirectory.Split (new char [] { ';' }, StringSplitOptions.RemoveEmptyEntries))
resolver.AddSearchDirectory (dir);

var topAssemblyReferences = new List<AssemblyDefinition> ();
var topAssemblyReferences = new List<string> (Assemblies.Length);
var logger = new NuGetLogger((s) => {
LogDebugMessage ("{0}", s);
});
Expand All @@ -91,33 +87,26 @@ void Execute (DirectoryAssemblyResolver resolver)

try {
foreach (var assembly in Assemblies) {
var assembly_path = Path.GetDirectoryName (assembly.ItemSpec);

if (!resolver.SearchDirectories.Contains (assembly_path))
resolver.SearchDirectories.Add (assembly_path);

// Add each user assembly and all referenced assemblies (recursive)
var assemblyDef = resolver.Load (assembly.ItemSpec);
if (assemblyDef == null)
throw new InvalidOperationException ("Failed to load assembly " + assembly.ItemSpec);
if (MonoAndroidHelper.IsReferenceAssembly (assemblyDef)) {
string assemblyPath = resolver.ResolveAssembly (assembly.ItemSpec);
if (MonoAndroidHelper.IsReferenceAssembly (assemblyPath)) {
// Resolve "runtime" library
var asmFullPath = Path.GetFullPath (assembly.ItemSpec);
if (lockFile != null)
assemblyDef = ResolveRuntimeAssemblyForReferenceAssembly (lockFile, resolver, asmFullPath);
if (lockFile == null || assemblyDef == null) {
LogCodedWarning ("XA0107", asmFullPath, 0, "Ignoring {0} as it is a Reference Assembly", asmFullPath);
assemblyPath = ResolveRuntimeAssemblyForReferenceAssembly (lockFile, assembly.ItemSpec);
if (lockFile == null || assemblyPath == null) {
LogCodedWarning ("XA0107", assemblyPath, 0, "Ignoring {0} as it is a Reference Assembly", assemblyPath);
continue;
}
}
topAssemblyReferences.Add (assemblyDef);
topAssemblyReferences.Add (assemblyPath);
var taskItem = new TaskItem (assembly) {
ItemSpec = Path.GetFullPath (assemblyDef.MainModule.FileName),
ItemSpec = Path.GetFullPath (assemblyPath),
};
if (string.IsNullOrEmpty (taskItem.GetMetadata ("ReferenceAssembly"))) {
taskItem.SetMetadata ("ReferenceAssembly", taskItem.ItemSpec);
}
assemblies [assemblyDef.Name.Name] = taskItem;
string assemblyName = Path.GetFileNameWithoutExtension (assemblyPath);
assemblies [assemblyName] = taskItem;
}
} catch (Exception ex) {
LogError ("Exception while loading assemblies: {0}", ex);
Expand Down Expand Up @@ -171,7 +160,7 @@ void Execute (DirectoryAssemblyResolver resolver)
readonly Dictionary<string, int> api_levels = new Dictionary<string, int> ();
int indent = 2;

AssemblyDefinition ResolveRuntimeAssemblyForReferenceAssembly (LockFile lockFile, DirectoryAssemblyResolver resolver, string assemblyPath)
string ResolveRuntimeAssemblyForReferenceAssembly (LockFile lockFile, string assemblyPath)
{
if (string.IsNullOrEmpty(TargetMoniker))
return null;
Expand Down Expand Up @@ -200,16 +189,16 @@ AssemblyDefinition ResolveRuntimeAssemblyForReferenceAssembly (LockFile lockFile
path = Path.Combine (folder.Path, libraryPath.Path, runtime.Path).Replace('/', Path.DirectorySeparatorChar);
if (!File.Exists (path))
continue;
LogDebugMessage ($"Attempting to load {path}");
return resolver.Load (path, forceLoad: true);
return path;
}
return null;
}

void AddAssemblyReferences (DirectoryAssemblyResolver resolver, Dictionary<string, ITaskItem> assemblies, AssemblyDefinition assembly, List<string> resolutionPath)
void AddAssemblyReferences (MetadataResolver resolver, Dictionary<string, ITaskItem> assemblies, string assemblyPath, List<string> resolutionPath)
{
var assemblyName = assembly.Name.Name;
var fullPath = Path.GetFullPath (assembly.MainModule.FileName);
var reader = resolver.GetAssemblyReader (assemblyPath);
var assembly = reader.GetAssemblyDefinition ();
var assemblyName = reader.GetString (assembly.Name);

// Don't repeat assemblies we've already done
bool topLevel = resolutionPath == null;
Expand All @@ -219,22 +208,23 @@ void AddAssemblyReferences (DirectoryAssemblyResolver resolver, Dictionary<strin
if (resolutionPath == null)
resolutionPath = new List<string>();

CheckAssemblyAttributes (assembly);
CheckAssemblyAttributes (assembly, reader);

LogMessage ("{0}Adding assembly reference for {1}, recursively...", new string (' ', indent), assembly.Name);
resolutionPath.Add (assembly.Name.Name);
LogMessage ("{0}Adding assembly reference for {1}, recursively...", new string (' ', indent), assemblyName);
resolutionPath.Add (assemblyName);
indent += 2;

// Add this assembly
if (!topLevel) {
assemblies [assemblyName] = CreateAssemblyTaskItem (fullPath);
assemblies [assemblyName] = CreateAssemblyTaskItem (Path.GetFullPath (assemblyPath));
}

// Recurse into each referenced assembly
foreach (AssemblyNameReference reference in assembly.MainModule.AssemblyReferences) {
AssemblyDefinition reference_assembly;
foreach (var handle in reader.AssemblyReferences) {
var reference = reader.GetAssemblyReference (handle);
string reference_assembly;
try {
reference_assembly = resolver.Resolve (reference);
reference_assembly = resolver.ResolveAssembly (reader.GetString (reference.Name));
} catch (FileNotFoundException ex) {
var references = new StringBuilder ();
for (int i = 0; i < resolutionPath.Count; i++) {
Expand All @@ -261,25 +251,30 @@ void AddAssemblyReferences (DirectoryAssemblyResolver resolver, Dictionary<strin
resolutionPath.RemoveAt (resolutionPath.Count - 1);
}

void CheckAssemblyAttributes (AssemblyDefinition assembly)
void CheckAssemblyAttributes (AssemblyDefinition assembly, MetadataReader reader)
{
foreach (var att in assembly.CustomAttributes) {
switch (att.AttributeType.FullName) {
foreach (var handle in assembly.GetCustomAttributes ()) {
var attribute = reader.GetCustomAttribute (handle);
switch (reader.GetCustomAttributeFullName (attribute)) {
case "Java.Interop.DoNotPackageAttribute": {
string file = (string)att.ConstructorArguments.First ().Value;
if (string.IsNullOrWhiteSpace (file))
LogError ("In referenced assembly {0}, Java.Interop.DoNotPackageAttribute requires non-null file name.", assembly.FullName);
do_not_package_atts.Add (Path.GetFileName (file));
var decoded = attribute.DecodeValue (DummyCustomAttributeProvider.Instance);
if (decoded.FixedArguments.Length > 0) {
string file = decoded.FixedArguments [0].Value?.ToString ();
if (string.IsNullOrWhiteSpace (file))
LogError ("In referenced assembly {0}, Java.Interop.DoNotPackageAttribute requires non-null file name.", assembly.GetAssemblyName ().FullName);
do_not_package_atts.Add (Path.GetFileName (file));
}
}
break;
case "System.Runtime.Versioning.TargetFrameworkAttribute": {
foreach (var p in att.ConstructorArguments) {
var value = p.Value.ToString ();
var decoded = attribute.DecodeValue (DummyCustomAttributeProvider.Instance);
foreach (var p in decoded.FixedArguments) {
var value = p.Value?.ToString ();
if (value.StartsWith ("MonoAndroid")) {
var values = value.Split ('=');
var apiLevel = MonoAndroidHelper.SupportedVersions.GetApiLevelFromFrameworkVersion (values [1]);
if (apiLevel != null) {
var assemblyName = assembly.Name.Name;
var assemblyName = reader.GetString (assembly.Name);
Log.LogDebugMessage ("{0}={1}", assemblyName, apiLevel);
api_levels [assemblyName] = apiLevel.Value;
}
Expand All @@ -305,7 +300,7 @@ static LinkModes ParseLinkMode (string linkmode)
return mode;
}

void AddI18nAssemblies (DirectoryAssemblyResolver resolver, Dictionary<string, ITaskItem> assemblies)
void AddI18nAssemblies (MetadataResolver resolver, Dictionary<string, ITaskItem> assemblies)
{
var i18n = Linker.ParseI18nAssemblies (I18nAssemblies);
var link = ParseLinkMode (LinkMode);
Expand All @@ -332,10 +327,10 @@ void AddI18nAssemblies (DirectoryAssemblyResolver resolver, Dictionary<string, I
ResolveI18nAssembly (resolver, "I18N.West", assemblies);
}

void ResolveI18nAssembly (DirectoryAssemblyResolver resolver, string name, Dictionary<string, ITaskItem> assemblies)
void ResolveI18nAssembly (MetadataResolver resolver, string name, Dictionary<string, ITaskItem> assemblies)
{
var assembly = resolver.Resolve (AssemblyNameReference.Parse (name));
var assemblyFullPath = Path.GetFullPath (assembly.MainModule.FileName);
var assembly = resolver.ResolveAssembly (name);
var assemblyFullPath = Path.GetFullPath (assembly);
assemblies [name] = CreateAssemblyTaskItem (assemblyFullPath);
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
using System.Reflection.Metadata;

namespace Xamarin.Android.Tasks
{
/// <summary>
/// A helper type for System.Reflection.Metadata. Getting the value of custom attribute arguments is a bit convoluted, if you merely want the values.
///
/// This interface allows usage such as:
/// CustomAttribute attribute = reader.GetCustomAttribute (handle);
/// CustomAttributeValue<object> decoded = attribute.DecodeValue (DummyCustomAttributeProvider.Instance);
/// </summary>
public class DummyCustomAttributeProvider : ICustomAttributeTypeProvider<object>
{
public static readonly DummyCustomAttributeProvider Instance = new DummyCustomAttributeProvider ();

public object GetPrimitiveType (PrimitiveTypeCode typeCode) => null;

public object GetSystemType () => null;

public object GetSZArrayType (object elementType) => null;

public object GetTypeFromDefinition (MetadataReader reader, TypeDefinitionHandle handle, byte rawTypeKind) => null;

public object GetTypeFromReference (MetadataReader reader, TypeReferenceHandle handle, byte rawTypeKind) => null;

public object GetTypeFromSerializedName (string name) => null;

public PrimitiveTypeCode GetUnderlyingEnumType (object type) => default (PrimitiveTypeCode);

public bool IsSystemType (object type) => false;
}
}
21 changes: 21 additions & 0 deletions src/Xamarin.Android.Build.Tasks/Utilities/MetadataExtensions.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
using System.Reflection.Metadata;

namespace Xamarin.Android.Tasks
{
public static class MetadataExtensions
{
public static string GetCustomAttributeFullName (this MetadataReader reader, CustomAttribute attribute)
{
if (attribute.Constructor.Kind == HandleKind.MemberReference) {
var ctor = reader.GetMemberReference ((MemberReferenceHandle)attribute.Constructor);
var type = reader.GetTypeReference ((TypeReferenceHandle)ctor.Parent);
return reader.GetString (type.Namespace) + "." + reader.GetString (type.Name);
} else if (attribute.Constructor.Kind == HandleKind.MethodDefinition) {
var ctor = reader.GetMethodDefinition ((MethodDefinitionHandle)attribute.Constructor);
var type = reader.GetTypeDefinition (ctor.GetDeclaringType ());
return reader.GetString (type.Namespace) + "." + reader.GetString (type.Name);
}
return null;
}
}
}
61 changes: 61 additions & 0 deletions src/Xamarin.Android.Build.Tasks/Utilities/MetadataResolver.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Reflection.Metadata;
using System.Reflection.PortableExecutable;

namespace Xamarin.Android.Tasks
{
/// <summary>
/// A replacement for DirectoryAssemblyResolver, using System.Reflection.Metadata
/// </summary>
public class MetadataResolver : IDisposable
{
readonly Dictionary<string, PEReader> cache = new Dictionary<string, PEReader> ();
readonly List<string> searchDirectories = new List<string> ();

public MetadataReader GetAssemblyReader (string assemblyName)
{
var key = Path.GetFileNameWithoutExtension (assemblyName);
if (!cache.TryGetValue (key, out PEReader reader)) {
var assemblyPath = ResolveAssembly (assemblyName);
cache.Add (key, reader = new PEReader (File.OpenRead (assemblyPath)));
}
return reader.GetMetadataReader ();
}

public void AddSearchDirectory (string directory)
{
directory = Path.GetFullPath (directory);
if (!searchDirectories.Contains (directory))
searchDirectories.Add (directory);
}

public string ResolveAssembly (string assemblyName)
{
string assemblyPath = assemblyName;
if (!assemblyPath.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) {
assemblyPath += ".dll";
}
if (File.Exists (assemblyPath)) {
AddSearchDirectory (Path.GetDirectoryName (assemblyPath));
return assemblyPath;
}
foreach (var dir in searchDirectories) {
var path = Path.Combine (dir, assemblyPath);
if (File.Exists (path))
return path;
}

throw new FileNotFoundException ($"Could not load assembly '{assemblyName}'.", assemblyName);
}

public void Dispose ()
{
foreach (var provider in cache.Values) {
provider.Dispose ();
}
cache.Clear ();
}
}
}
27 changes: 13 additions & 14 deletions src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,18 @@
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.IO;
using System.Reflection.Metadata;
using System.Reflection.PortableExecutable;
using System.Security.Cryptography;
using Mono.Security.Cryptography;
using Xamarin.Android.Tools;
using Xamarin.Tools.Zip;
using Mono.Cecil;

#if MSBUILD
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;
#endif

using Xamarin.Android.Tools;

namespace Xamarin.Android.Tasks
{
public class MonoAndroidHelper
Expand Down Expand Up @@ -307,16 +304,18 @@ public static bool IsFrameworkAssembly (string assembly, bool checkSdkPath)

public static bool IsReferenceAssembly (string assembly)
{
var rp = new ReaderParameters { ReadSymbols = false };
using (var a = AssemblyDefinition.ReadAssembly (assembly, rp))
return IsReferenceAssembly (a);
}

public static bool IsReferenceAssembly (AssemblyDefinition assembly)
{
if (!assembly.HasCustomAttributes)
using (var stream = File.OpenRead (assembly))
using (var pe = new PEReader (stream)) {
var reader = pe.GetMetadataReader ();
var assemblyDefinition = reader.GetAssemblyDefinition ();
foreach (var handle in assemblyDefinition.GetCustomAttributes ()) {
var attribute = reader.GetCustomAttribute (handle);
var attributeName = reader.GetCustomAttributeFullName (attribute);
if (attributeName == "System.Runtime.CompilerServices.ReferenceAssemblyAttribute")
return true;
}
return false;
return assembly.CustomAttributes.Any (t => t.AttributeType.FullName == "System.Runtime.CompilerServices.ReferenceAssemblyAttribute");
}
}

public static bool ExistsInFrameworkPath (string assembly)
Expand Down
Loading

0 comments on commit 2dfcac4

Please sign in to comment.