Skip to content

Commit

Permalink
Use ILLink to be smarter about using P/Invoke overrides (#1645)
Browse files Browse the repository at this point in the history
* Build iOS in the native build + move to fat bins

* Autogenerate ILLink substitutions

* Integrate SilkTouch runtime config + add iOS back to TriangleNET6

* Update src/Core/Silk.NET.SilkTouch/NativeApiGenerator.cs

* Update SDL2 binaries (#1647)

* New binaries for SDL2 on Microsoft Windows 10.0.20348

* New binaries for SDL2 on Darwin 21.6.0 Darwin Kernel Version 21.6.0: Thu Jul  6 22:18:26 PDT 2023; root:xnu-8020.240.18.702.13~1/RELEASE_X86_64

---------

Co-authored-by: The Silk.NET Automaton <9011267+dotnet-bot@users.noreply.github.com>

* Successfully wire up the linker (it works!)

* Get iOS building again

* Fix packaging

* Split ILLink.xml and .targets generation into two tasks

* Try and get a SourceLink win

* Update Build.NuGet.cs

Co-authored-by: Kai Jellinghaus <contact@kaij.tech>

* Targets file was ignored for some reason?

---------

Co-authored-by: silkdotnet <85832961+silkdotnet@users.noreply.github.com>
Co-authored-by: The Silk.NET Automaton <9011267+dotnet-bot@users.noreply.github.com>
Co-authored-by: Kai Jellinghaus <contact@kaij.tech>
  • Loading branch information
4 people authored Sep 17, 2023
1 parent 37e3c39 commit ef2a386
Show file tree
Hide file tree
Showing 50 changed files with 699 additions and 166 deletions.
8 changes: 5 additions & 3 deletions .nuke/build.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,6 @@
"items": {
"type": "string",
"enum": [
"Angle",
"Assimp",
"BuildLibSilkDroid",
"Clean",
Expand All @@ -161,9 +160,11 @@
"RegenerateBindings",
"Restore",
"SDL2",
"Shaderc",
"ShipApi",
"SignPackages",
"Sln",
"SPIRVCross",
"SPIRVReflect",
"SwiftShader",
"Test",
Expand All @@ -184,7 +185,6 @@
"items": {
"type": "string",
"enum": [
"Angle",
"Assimp",
"BuildLibSilkDroid",
"Clean",
Expand All @@ -203,9 +203,11 @@
"RegenerateBindings",
"Restore",
"SDL2",
"Shaderc",
"ShipApi",
"SignPackages",
"Sln",
"SPIRVCross",
"SPIRVReflect",
"SwiftShader",
"Test",
Expand Down Expand Up @@ -233,4 +235,4 @@
}
}
}
}
}
6 changes: 5 additions & 1 deletion build/nuke/Build.NuGet.cs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ partial class Build
static IEnumerable<string> Packages => Directory.GetFiles(PackageDirectory, "*.nupkg")
.Where(x => Path.GetFileName(x).StartsWith("Silk.NET") || Path.GetFileName(x).StartsWith("Ultz.Native"));

static IEnumerable<string> SymbolPackages => Directory.GetFiles(PackageDirectory, "*.snupkg")
.Where(x => Path.GetFileName(x).StartsWith("Silk.NET") || Path.GetFileName(x).StartsWith("Ultz.Native"));

Target PushToNuGet => CommonTarget
(
x => x.DependsOn(Pack)
Expand All @@ -35,7 +38,8 @@ async Task<IEnumerable<Output>> PushPackages()
var outputs = Enumerable.Empty<Output>();
const int rateLimit = 300;

var allFiles = Packages.Select((x, i) => new { Index = i, Value = x })
var allFiles = Packages.Concat(SymbolPackages)
.Select((x, i) => new { Index = i, Value = x })
.GroupBy(x => x.Index / rateLimit)
.Select(x => x.Select(v => v.Value).ToList())
.ToList();
Expand Down
2 changes: 1 addition & 1 deletion build/nuke/Build.Packaging.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ partial class Build
(
x => x.DependsOn(Restore)
.After(Clean, RegenerateBindings, BuildLibSilkDroid)
.Produces("build/output_packages/*.nupkg")
.Produces("build/output_packages/*.nupkg", "build/output_packages/*.snupkg")
.Executes
(
() => ErrorsOnly<DotNetPackSettings>
Expand Down
17 changes: 10 additions & 7 deletions build/nuke/Build.Support.cs
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ partial class Build

[Parameter("Outputs build warnings instead of keeping the MSBuild logging quiet with just errors.")]
bool Warnings;

static int IndexOfOrThrow(string x, char y)
{
var idx = x.IndexOf(y);
Expand Down Expand Up @@ -116,7 +116,10 @@ Dictionary<string, object> ProcessedMsbuildProperties
{
try
{
DotNet("dotnet nuget remove source \"Silk-PushPackages\"");
if (DotNet("dotnet nuget list source").Any(x => x.Text.Contains("Silk-PushPackages")))
{
DotNet("dotnet nuget remove source \"Silk-PushPackages\"");
}
}
catch
{
Expand All @@ -127,9 +130,9 @@ Dictionary<string, object> ProcessedMsbuildProperties
);

AbsolutePath SourceDirectory => RootDirectory / "src";

ConcurrentDictionary<Target, Target> Targets = new();
static Target GetEmptyTarget() => _ => _.Executes(() => {});
static Target GetEmptyTarget() => _ => _.Executes(() => { });
Target CommonTarget([CanBeNull] Target actualTarget = null) => Targets.GetOrAdd
(
actualTarget ??= GetEmptyTarget(), def =>
Expand All @@ -147,7 +150,7 @@ async Task AddOrUpdatePrComment(string type, string file, bool editOnly = false,
Logger.Info("GitHub token not found, skipping writing a comment.");
return;
}

var @ref = GitHubActions.Instance?.Ref;
if (string.IsNullOrWhiteSpace(@ref))
{
Expand All @@ -167,7 +170,7 @@ async Task AddOrUpdatePrComment(string type, string file, bool editOnly = false,
Logger.Info($"Couldn't parse {@prMatch.Groups[1].Value} as an int, skipping writing a comment.");
return;
}

var github = new GitHubClient
(
new ProductHeaderValue("Silk.NET-CI"),
Expand All @@ -189,7 +192,7 @@ async Task AddOrUpdatePrComment(string type, string file, bool editOnly = false,
{
commentText = commentText.Replace($"{{{key}}}", value);
}

commentText = commentText.Replace("{actionsRun}", GitHubActions.Instance?.RunNumber.ToString())
.Replace("{typeId}", type);

Expand Down
6 changes: 5 additions & 1 deletion build/nuke/Native/Core.cs
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,12 @@ partial class Build {
[CanBeNull] string AndroidHomeValue;

static string JobsArg => string.IsNullOrWhiteSpace(GitHubActions.Instance?.Job)
? $" -j{Environment.ProcessorCount}"
? $" -j{Jobs}"
: string.Empty;

static int Jobs => string.IsNullOrWhiteSpace(GitHubActions.Instance?.Job)
? Environment.ProcessorCount - 1
: 1;

public void CopyAs(AbsolutePath @out, string from, string to)
{
Expand Down
76 changes: 56 additions & 20 deletions build/nuke/Native/SDL2.cs
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,29 @@
using static Nuke.Common.Tools.Git.GitTasks;
using static Nuke.Common.Tools.GitHub.GitHubTasks;

partial class Build {
partial class Build
{
AbsolutePath SDL2Path => RootDirectory / "build" / "submodules" / "SDL";

static readonly (string Sdk, string Arch, string Rid, string Args)[] iPhoneConfigs = new[]
{
// NB: Xcode has forced us to stop support for i386 archs
// NB: Xcode has forced us to stop support for iPhone 5 and below (armv7, armv7s)
("iphonesimulator", "x86_64", "iossimulator", "-target \"Static Library-iOS\""),
("iphonesimulator", "arm64", "iossimulator", "-target \"Static Library-iOS\""),
("iphoneos", "arm64", "ios", "-target \"Static Library-iOS\""),
("iphoneos", "arm64e", "ios", "-target \"Static Library-iOS\""),
("appletvsimulator", "x86_64", "tvossimulator", "-target \"Static Library-tvOS\""),
("appletvsimulator", "arm64", "tvossimulator", "-target \"Static Library-tvOS\""),
("appletvos", "arm64", "tvos", "-target \"Static Library-tvOS\""),
("appletvos", "arm64e", "tvos", "-target \"Static Library-tvOS\""),
("macosx", "x86_64", "osx", "-target \"Shared Library\""),
("macosx", "arm64", "osx", "-target \"Shared Library\""),
// TODO figure out how to compile for Mac Catalyst
//("macosx", "x86_64", "maccatalyst", "-target \"Shared Library-iOS\" -destination \"platform=macOS,variant=Mac Catalyst\""),
//("macosx", "arm64", "maccatalyst", "-target \"Shared Library-iOS\" -destination \"platform=macOS,variant=Mac Catalyst\""),
};

Target SDL2 => CommonTarget
(
x => x.Before(Compile)
Expand All @@ -45,7 +65,7 @@ partial class Build {
EnsureCleanDirectory(x64BuildDir);
EnsureCleanDirectory(ARM64BuildDir);
if(OperatingSystem.IsWindows())
if (OperatingSystem.IsWindows())
{
var prepare = "cmake .. -DBUILD_SHARED_LIBS=ON";
var build = $"cmake --build . --config Release{JobsArg}";
Expand All @@ -64,16 +84,16 @@ partial class Build {
CopyFile(ARM64BuildDir / "Release" / "SDL2.dll", runtimes / "win-arm64" / "native" / "SDL2.dll", FileExistsPolicy.Overwrite);
}
if(OperatingSystem.IsLinux())
if (OperatingSystem.IsLinux())
{
if(RuntimeInformation.OSArchitecture == Architecture.Arm64)
if (RuntimeInformation.OSArchitecture == Architecture.Arm64)
{
InheritedShell("cmake ..", x86BuildDir).AssertZeroExitCode();
InheritedShell("cmake --build .", x86BuildDir).AssertZeroExitCode();
CopyFile(ARM64BuildDir.GlobFiles("libSDL2-2.0.so*").First(), runtimes / "linux-arm64" / "native" / "libSDL2-2.0.so", FileExistsPolicy.Overwrite);
}
else if (RuntimeInformation.OSArchitecture == Architecture.X64)
}
else if (RuntimeInformation.OSArchitecture == Architecture.X64)
{
var envVars32bit = "CFLAGS='-m32 -O2' CXXFLAGS='-m32 -O2' LDFLAGS=-m32";
var envVars64bit = "CFLAGS=-O2 CXXFLAGS=-O2";
Expand All @@ -92,26 +112,42 @@ partial class Build {
CopyFile((x86BuildDir / "lib").GlobFiles("libSDL2-2.0.so*").First(), runtimes / "linux-x86" / "native" / "libSDL2-2.0.so", FileExistsPolicy.Overwrite);
CopyFile((x64BuildDir / "lib").GlobFiles("libSDL2-2.0.so*").First(), runtimes / "linux-x64" / "native" / "libSDL2-2.0.so", FileExistsPolicy.Overwrite);
}
else
}
else
{
throw new Exception($"Unable to build SDL libs on your architecture ({RuntimeInformation.OSArchitecture}).");
}
}
if(OperatingSystem.IsMacOS())
if (OperatingSystem.IsMacOS())
{
var prepare = "cmake .. -DBUILD_SHARED_LIBS=ON";
var build = $"cmake --build . --config Release{JobsArg}";
InheritedShell($"{prepare} -DCMAKE_OSX_ARCHITECTURES=x86_64", x64BuildDir).AssertZeroExitCode();
InheritedShell(build, x64BuildDir).AssertZeroExitCode();
InheritedShell($"{prepare} -DCMAKE_OSX_ARCHITECTURES=arm64", ARM64BuildDir).AssertZeroExitCode();
InheritedShell(build, ARM64BuildDir).AssertZeroExitCode();
CopyAs(x64BuildDir, "**/*.dylib", runtimes / "osx-x64" / "native" / "libSDL2-2.0.dylib");
CopyAs(ARM64BuildDir, "**/*.dylib", runtimes / "osx-arm64" / "native" / "libSDL2-2.0.dylib");
// iOS build/hackery ported from https://github.com/Ultz/SDL-Xamarin.iOS
var mainH = File.ReadAllText(SDL2Path / "include" / "SDL_main.h");
try
{
File.WriteAllText(SDL2Path / "include" / "SDL_main.h", $"#define SDL_MAIN_HANDLED\n{mainH}");
EnsureCleanDirectory(SDL2Path / "allbuild");
foreach (var (sdk, arch, rid, args) in iPhoneConfigs)
{
InheritedShell($"xcodebuild -project SDL.xcodeproj {args} -sdk {sdk} -arch {arch} -configuration Release clean build -jobs {Jobs}", SDL2Path / "XCode" / "SDL")
.AssertZeroExitCode();
var ext = rid is "maccatalyst" or "osx" ? "dylib" : "a";
var cfg = rid is "maccatalyst" or "osx" ? "Release" : $"Release-{sdk}";
CopyFile(SDL2Path / "XCode" / "SDL" / "build" / cfg / $"libSDL2.{ext}", SDL2Path / "allbuild" / $"libSDL2.{sdk}.{arch}.{rid}.{ext}");
}
foreach (var rid in iPhoneConfigs.GroupBy(x => x.Rid))
{
var ext = rid.Key is "maccatalyst" or "osx" ? "dylib" : "a";
var @in = string.Join("\" \"", rid.Select(x => SDL2Path / "allbuild" / $"libSDL2.{x.Sdk}.{x.Arch}.{rid.Key}.{ext}"));
var @out = runtimes / rid.Key / "native" / (ext is "dylib" ? "libSDL2-2.0.dylib" : "libSDL2.a");
EnsureCleanDirectory(Path.GetDirectoryName(@out));
InheritedShell($"lipo -create -output \"{@out}\" \"{@in}\"").AssertZeroExitCode();
}
}
finally
{
File.WriteAllText(SDL2Path / "include" / "SDL_main.h", mainH);
}
}
PrUpdatedNativeBinary("SDL2");
Expand Down
55 changes: 54 additions & 1 deletion build/props/bindings.props
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
<Project>
<Project InitialTargets="SilkGenerateILLinkSubs">
<ItemGroup>
<ProjectReference Include="$(MSBuildThisFileDirectory)..\..\src\Core\Silk.NET.Core\Silk.NET.Core.csproj" />
<ProjectReference Include="$(MSBuildThisFileDirectory)..\..\src\Core\Silk.NET.SilkTouch\Silk.NET.SilkTouch.csproj" OutputItemType="Analyzer" ReferenceOutputAssembly="false"/>
Expand All @@ -7,4 +7,57 @@
<SilkPublicApiExempt>true</SilkPublicApiExempt>
</PropertyGroup>
<Import Project="common.props" />
<PropertyGroup>
<TargetsForTfmSpecificContentInPackage>$(TargetsForTfmSpecificContentInPackage);SilkGenerateILLinkTargets</TargetsForTfmSpecificContentInPackage>
</PropertyGroup>
<Target Name="SilkGenerateILLinkTargets" Condition="$(TargetFramework.Contains('net6')) or $(TargetFramework.Contains('net7')) or $(TargetFramework.Contains('net8'))">
<GetNuGetShortFolderName TargetFrameworkMoniker="$(TargetFrameworkMoniker)" TargetPlatformMoniker="$(TargetPlatformMoniker)">
<Output TaskParameter="NuGetShortFolderName" PropertyName="_SilkNuGetShortFolderName" />
</GetNuGetShortFolderName>
<PropertyGroup>
<SilkGeneratedTargetPath>$(IntermediateOutputPath)$(TargetFramework)/$(PackageId).targets</SilkGeneratedTargetPath>
<SilkGeneratedPropPrefix>_$([System.String]::Copy('$(PackageId)').Replace('.', '_'))_</SilkGeneratedPropPrefix>
</PropertyGroup>
<ItemGroup Condition="'@(SilkPInvokeOverride)' != ''">
<SilkGeneratedTargetLine Include="&lt;Project InitialTargets=&quot;$(SilkGeneratedPropPrefix)ConfigureSilkTouchRuntime&quot;&gt;" />
<SilkGeneratedTargetLine Include="&lt;Target Name=&quot;$(SilkGeneratedPropPrefix)ConfigureSilkTouchRuntime&quot;&gt;" />
<SilkGeneratedTargetLine Include="&lt;PropertyGroup&gt;" />
<SilkGeneratedTargetLine Include="&lt;$(SilkGeneratedPropPrefix)ExternalPInvokes&gt;%40(SilkExternalPInvoke)&lt;/$(SilkGeneratedPropPrefix)ExternalPInvokes&gt;" />
<SilkGeneratedTargetLine Include="&lt;$(SilkGeneratedPropPrefix)InternalPInvokes&gt;%40(SilkInternalPInvoke)&lt;/$(SilkGeneratedPropPrefix)InternalPInvokes&gt;" />
<SilkGeneratedTargetLine Include="&lt;$(SilkGeneratedPropPrefix)Opts&gt;%40(RuntimeHostConfigurationOption)&lt;/$(SilkGeneratedPropPrefix)Opts&gt;" />
<SilkGeneratedTargetLine Include="&lt;/PropertyGroup&gt;" />
<SilkGeneratedTargetLine Include="&lt;ItemGroup&gt;" />
<SilkGeneratedTargetLine Include="&lt;RuntimeHostConfigurationOption Condition=&quot;(!%24([System.String]::Copy('%24($(SilkGeneratedPropPrefix)ExternalPInvokes)').Contains('%(SilkPInvokeOverride.SilkClass)')) and !%24([System.String]::Copy('%24($(SilkGeneratedPropPrefix)Opts)').Contains('$([System.String]::Copy('%(SilkPInvokeOverride.SilkClass)').Replace('.', '_').ToUpper())_ENABLE_PINVOKE_OVERRIDE_%(SilkPInvokeOverride.Identity)'))) and (('%24(SilkEnableStaticLinking)' == 'true') or %24([System.String]::Copy('%24($(SilkGeneratedPropPrefix)InternalPInvokes)').ToLower().Contains('%(SilkPInvokeOverride.SilkClass)')) or %(SilkPInvokeOverride.DownstreamCondition))&quot; Include=&quot;$([System.String]::Copy('%(SilkPInvokeOverride.SilkClass)').Replace('.', '_').ToUpper())_ENABLE_PINVOKE_OVERRIDE_%(SilkPInvokeOverride.Identity)&quot; Value=&quot;true&quot; Trim=&quot;true&quot; /&gt;" />
<SilkGeneratedTargetLine Include="&lt;RuntimeHostConfigurationOption Condition=&quot;(!%24([System.String]::Copy('%24($(SilkGeneratedPropPrefix)Opts)').Contains('$([System.String]::Copy('%(SilkPInvokeOverride.SilkClass)').Replace('.', '_').ToUpper())_ENABLE_PINVOKE_OVERRIDE_%(SilkPInvokeOverride.Identity)'))) and (!(('%24(SilkEnableStaticLinking)' == 'true') or %24([System.String]::Copy('%24($(SilkGeneratedPropPrefix)InternalPInvokes)').ToLower().Contains('%(SilkPInvokeOverride.SilkClass)')) or %(SilkPInvokeOverride.DownstreamCondition)))&quot; Include=&quot;$([System.String]::Copy('%(SilkPInvokeOverride.SilkClass)').Replace('.', '_').ToUpper())_ENABLE_PINVOKE_OVERRIDE_%(SilkPInvokeOverride.Identity)&quot; Value=&quot;false&quot; Trim=&quot;true&quot; /&gt;" />
<SilkGeneratedTargetLine Include="&lt;/ItemGroup&gt;" />
<SilkGeneratedTargetLine Include="&lt;/Target&gt;" />
<SilkGeneratedTargetLine Include="&lt;/Project&gt;" />
<SilkGeneratedTargetFile Include="$(SilkGeneratedTargetPath)" />
</ItemGroup>
<WriteLinesToFile File="@(SilkGeneratedTargetFile)" Lines="@(SilkGeneratedTargetLine)" Overwrite="true" Encoding="UTF-8" Condition="'@(SilkPInvokeOverride)' != ''" />
<ItemGroup Condition="'@(SilkPInvokeOverride)' != ''">
<TfmSpecificPackageFile Include="@(SilkGeneratedTargetFile)" Link="build/$(_SilkNuGetShortFolderName)/$(PackageId).targets" Pack="true" PackagePath="build/$(_SilkNuGetShortFolderName)/$(PackageId).targets" />
</ItemGroup>
</Target>
<Target Name="SilkGenerateILLinkSubs" Condition="$(TargetFramework.Contains('net6')) or $(TargetFramework.Contains('net7')) or $(TargetFramework.Contains('net8'))">
<PropertyGroup>
<SilkGeneratedILLinkPath>$(IntermediateOutputPath)$(TargetFramework)/ILLink.Substitutions.xml</SilkGeneratedILLinkPath>
</PropertyGroup>
<ItemGroup Condition="'@(SilkPInvokeOverride)' != ''">
<SilkGeneratedILLinkLine Include="&lt;linker&gt;" />
<SilkGeneratedILLinkLine Include="&lt;assembly fullname=&quot;$(PackageId)&quot;&gt;" />
<SilkGeneratedILLinkLine Include="&lt;type fullname=&quot;SilkTouchRuntimeConfiguration&quot;&gt;" />
<SilkGeneratedILLinkLine Include="&lt;method signature=&quot;System.Boolean get_HasLinkTimeSubstitutions()&quot; body=&quot;stub&quot; value=&quot;true&quot; /&gt;" />
<SilkGeneratedILLinkLine Include="&lt;method signature=&quot;System.Boolean get_$([System.String]::Copy('%(SilkPInvokeOverride.SilkClass)').Replace('.', ''))PInvokeOverride%(SilkPInvokeOverride.Identity)()&quot; body=&quot;stub&quot; value=&quot;true&quot; feature=&quot;$([System.String]::Copy('%(SilkPInvokeOverride.SilkClass)').Replace('.', '_').ToUpper())_ENABLE_PINVOKE_OVERRIDE_%(SilkPInvokeOverride.Identity)&quot; featurevalue=&quot;true&quot; /&gt;" />
<SilkGeneratedILLinkLine Include="&lt;method signature=&quot;System.Boolean get_$([System.String]::Copy('%(SilkPInvokeOverride.SilkClass)').Replace('.', ''))PInvokeOverride%(SilkPInvokeOverride.Identity)()&quot; body=&quot;stub&quot; value=&quot;false&quot; feature=&quot;$([System.String]::Copy('%(SilkPInvokeOverride.SilkClass)').Replace('.', '_').ToUpper())_ENABLE_PINVOKE_OVERRIDE_%(SilkPInvokeOverride.Identity)&quot; featurevalue=&quot;false&quot; /&gt;" />
<SilkGeneratedILLinkLine Include="&lt;/type&gt;" />
<SilkGeneratedILLinkLine Include="&lt;/assembly&gt;" />
<SilkGeneratedILLinkLine Include="&lt;/linker&gt;" />
<SilkGeneratedILLinkFile Include="$(SilkGeneratedILLinkPath)" />
</ItemGroup>
<WriteLinesToFile File="@(SilkGeneratedILLinkFile)" Lines="@(SilkGeneratedILLinkLine)" Overwrite="true" Encoding="UTF-8" Condition="'@(SilkPInvokeOverride)' != ''" />
<ItemGroup Condition="'@(SilkPInvokeOverride)' != ''">
<EmbeddedResource Include="@(SilkGeneratedILLinkFile)" Link="Properties/ILLink.Substitutions.xml" LogicalName="ILLink.Substitutions.xml" />
</ItemGroup>
</Target>
</Project>
Loading

0 comments on commit ef2a386

Please sign in to comment.