From d9bec7ab787bf6ce529e774b5dd6d88e2c34001f Mon Sep 17 00:00:00 2001 From: Dylan Perks <11160611+Perksey@users.noreply.github.com> Date: Tue, 15 Mar 2022 18:15:42 +0000 Subject: [PATCH] This is the Silk.NET March 2022 Update (v2.14) (#836) * Upgrade Assimp, regenerate bindings, attempt to update assimp natives * Add workflow for assimp * Actually make the assimp workflow run * Revert SilkTouch debug change * PR the updated binary * Fix workflows(?), bin paths * Add back the wildcards, they're not doing much harm * Fix Linux paths * Mac versioning scheme in Core.Loader, fix PR opening * Fix CI * Attempt to fix various native workflows * Formatting * Fix weird merge conflicts * Update Assimp binaries (#839) * New binaries for Assimp on Darwin 20.6.0 Darwin Kernel Version 20.6.0: Wed Jan 12 22:22:42 PST 2022; root:xnu-7195.141.19~2/RELEASE_X86_64 * New binaries for Assimp on Darwin 20.6.0 Darwin Kernel Version 20.6.0: Wed Jan 12 22:22:42 PST 2022; root:xnu-7195.141.19~2/RELEASE_X86_64 * New binaries for Assimp on Darwin 20.6.0 Darwin Kernel Version 20.6.0: Wed Jan 12 22:22:42 PST 2022; root:xnu-7195.141.19~2/RELEASE_X86_64 * New binaries for Assimp on Darwin 20.6.0 Darwin Kernel Version 20.6.0: Wed Jan 12 22:22:42 PST 2022; root:xnu-7195.141.19~2/RELEASE_X86_64 Co-authored-by: runner Co-authored-by: runner Co-authored-by: The Silk.NET Automaton <9011267+dotnet-bot@users.noreply.github.com> * Update SwiftShader binaries (#838) * New binaries for SwiftShader on Microsoft Windows 10.0.20348 * New binaries for SwiftShader on Microsoft Windows 10.0.20348 * New binaries for SwiftShader on Microsoft Windows 10.0.20348 * New binaries for SwiftShader on Microsoft Windows 10.0.20348 * New binaries for SwiftShader on Microsoft Windows 10.0.20348 * New binaries for SwiftShader on Microsoft Windows 10.0.20348 * New binaries for SwiftShader on Microsoft Windows 10.0.20348 * New binaries for SwiftShader on Microsoft Windows 10.0.20348 * New binaries for SwiftShader on Microsoft Windows 10.0.20348 * New binaries for SwiftShader on Microsoft Windows 10.0.20348 Co-authored-by: The Silk.NET Automaton <9011267+dotnet-bot@users.noreply.github.com> * Update Vulkan Loader binaries (#837) * New binaries for Vulkan Loader on Microsoft Windows 10.0.20348 * New binaries for Vulkan Loader on Microsoft Windows 10.0.20348 * New binaries for Vulkan Loader on Microsoft Windows 10.0.20348 * New binaries for Vulkan Loader on Microsoft Windows 10.0.20348 * New binaries for Vulkan Loader on Microsoft Windows 10.0.20348 * New binaries for Vulkan Loader on Microsoft Windows 10.0.20348 * New binaries for Vulkan Loader on Microsoft Windows 10.0.20348 * New binaries for Vulkan Loader on Microsoft Windows 10.0.20348 * New binaries for Vulkan Loader on Microsoft Windows 10.0.20348 * New binaries for Vulkan Loader on Microsoft Windows 10.0.20348 Co-authored-by: The Silk.NET Automaton <9011267+dotnet-bot@users.noreply.github.com> * Update Build.Native.cs * New binaries for SwiftShader on Microsoft Windows 10.0.20348 (#842) Co-authored-by: The Silk.NET Automaton <9011267+dotnet-bot@users.noreply.github.com> * Update Assimp binaries (#841) * New binaries for Assimp on Linux 5.11.0-1028-azure #31~20.04.2-Ubuntu SMP Tue Jan 18 08:46:15 UTC 2022 * New binaries for Assimp on Microsoft Windows 10.0.20348 Co-authored-by: The Silk.NET Automaton <9011267+dotnet-bot@users.noreply.github.com> * New binaries for Vulkan Loader on Microsoft Windows 10.0.20348 (#840) Co-authored-by: The Silk.NET Automaton <9011267+dotnet-bot@users.noreply.github.com> * Regenerate bindings, update patch notes * New binaries for ANGLE on Darwin 20.6.0 Darwin Kernel Version 20.6.0: Wed Jan 12 22:22:42 PST 2022; root:xnu-7195.141.19~2/RELEASE_X86_64 (#844) Co-authored-by: The Silk.NET Automaton <9011267+dotnet-bot@users.noreply.github.com> * New binaries for ANGLE on Linux 5.11.0-1028-azure #31~20.04.2-Ubuntu SMP Tue Jan 18 08:46:15 UTC 2022 (#845) Co-authored-by: The Silk.NET Automaton <9011267+dotnet-bot@users.noreply.github.com> Co-authored-by: silkdotnet <85832961+silkdotnet@users.noreply.github.com> Co-authored-by: runner Co-authored-by: runner Co-authored-by: The Silk.NET Automaton <9011267+dotnet-bot@users.noreply.github.com> --- .nuke/build.schema.json | 6 + .../Silk.NET.NUKE/Build.Core.cs | 2 +- .../Silk.NET.NUKE/Build.Native.cs | 134 +++++++++++++++--- .../Silk.NET.NUKE/Build.Packaging.cs | 2 +- .../Silk.NET.NUKE/Build.ReviewHelpers.cs | 6 +- .../Silk.NET.NUKE/Build.Support.cs | 41 +++++- 6 files changed, 162 insertions(+), 29 deletions(-) diff --git a/.nuke/build.schema.json b/.nuke/build.schema.json index f5adb748ac..d4bb4e04bb 100644 --- a/.nuke/build.schema.json +++ b/.nuke/build.schema.json @@ -118,6 +118,7 @@ "type": "string", "enum": [ "Angle", + "Assimp", "BuildLibSilkDroid", "Clean", "Compile", @@ -153,6 +154,7 @@ "type": "string", "enum": [ "Angle", + "Assimp", "BuildLibSilkDroid", "Clean", "Compile", @@ -186,6 +188,10 @@ "Quiet", "Verbose" ] + }, + "Warnings": { + "type": "boolean", + "description": "Outputs build warnings instead of keeping the MSBuild logging quiet with just errors" } } } diff --git a/src/infrastructure/Silk.NET.NUKE/Build.Core.cs b/src/infrastructure/Silk.NET.NUKE/Build.Core.cs index e6d570e44f..819bdd83a1 100644 --- a/src/infrastructure/Silk.NET.NUKE/Build.Core.cs +++ b/src/infrastructure/Silk.NET.NUKE/Build.Core.cs @@ -84,7 +84,7 @@ partial class Build .After(Clean) .Executes ( - () => DotNetBuild + () => ErrorsOnly ( s => s.SetProjectFile(Solution) .SetConfiguration(Configuration) diff --git a/src/infrastructure/Silk.NET.NUKE/Build.Native.cs b/src/infrastructure/Silk.NET.NUKE/Build.Native.cs index 020acaebf0..33d544cee2 100644 --- a/src/infrastructure/Silk.NET.NUKE/Build.Native.cs +++ b/src/infrastructure/Silk.NET.NUKE/Build.Native.cs @@ -31,6 +31,10 @@ partial class Build [CanBeNull] string AndroidHomeValue; + static string JobsArg => string.IsNullOrWhiteSpace(GitHubActions.Instance?.GitHubJob) + ? $" -j{Environment.ProcessorCount}" + : string.Empty; + string AndroidHome { get @@ -122,10 +126,7 @@ string AndroidHome Git("checkout HEAD build/", SwiftShaderBuildPath / ".."); StartProcess("cmake", ".. -DCMAKE_BUILD_TYPE=Release", SwiftShaderBuildPath) .AssertZeroExitCode(); - var nonGitHubActionsArgs = string.IsNullOrWhiteSpace(GitHubActions.Instance.GitHubJob) - ? " --parallel" - : string.Empty; - StartProcess("cmake", $"--build .{nonGitHubActionsArgs} --config Release", SwiftShaderBuildPath) + StartProcess("cmake", $"--build .{JobsArg} --config Release", SwiftShaderBuildPath) .AssertWaitForExit(); // might fail... as long as the output exists we're happy var fname = sysName switch { @@ -275,7 +276,9 @@ string AndroidHome } ) ); + AbsolutePath GLFWPath => RootDirectory / "build" / "submodules" / "GLFW"; + Target GLFW => CommonTarget ( x => x.Before(Compile) @@ -286,7 +289,7 @@ string AndroidHome { var @out = GLFWPath / "build"; var prepare = "cmake -S. -B build -D BUILD_SHARED_LIBS=ON"; - var build = "cmake --build build --config Release"; + var build = $"cmake --build build --config Release{JobsArg}"; EnsureCleanDirectory(@out); var runtimes = RootDirectory / "src" / "Native" / "Silk.NET.GLFW.Native" / "runtimes"; if (OperatingSystem.IsWindows()) @@ -296,14 +299,14 @@ string AndroidHome InheritedShell(build, GLFWPath) .AssertZeroExitCode(); CopyAll(@out.GlobFiles("src/Release/glfw3.dll"), runtimes / "win-x64" / "native"); - + EnsureCleanDirectory(@out); - + InheritedShell($"{prepare} -A Win32", GLFWPath) .AssertZeroExitCode(); InheritedShell(build, GLFWPath) .AssertZeroExitCode(); - + CopyAll(@out.GlobFiles("src/Release/glfw3.dll"), runtimes / "win-x86" / "native"); } else if (OperatingSystem.IsLinux()) @@ -323,19 +326,22 @@ string AndroidHome CopyAll(@out.GlobFiles("src/libglfw.3.dylib"), runtimes / "osx-x64" / "native"); EnsureCleanDirectory(@out); - + InheritedShell($"{prepare} -DCMAKE_OSX_ARCHITECTURES=arm64", GLFWPath) .AssertZeroExitCode(); InheritedShell(build, GLFWPath) .AssertZeroExitCode(); - + CopyAll(@out.GlobFiles("src/libglfw.3.dylib"), runtimes / "osx-arm64" / "native"); } + + PrUpdatedNativeBinary("GLFW"); } ) ); AbsolutePath VulkanLoaderPath => RootDirectory / "build" / "submodules" / "Vulkan-Loader"; + Target VulkanLoader => CommonTarget ( x => x.Before(Compile) @@ -348,11 +354,12 @@ string AndroidHome EnsureCleanDirectory(@out); var abi = OperatingSystem.IsWindows() ? " -DCMAKE_GENERATOR_PLATFORM=Win32" : string.Empty; InheritedShell - ( - $"cmake -S. -Bbuild -DUPDATE_DEPS=On -DCMAKE_BUILD_TYPE=Release{abi}", - VulkanLoaderPath - ).AssertZeroExitCode(); - InheritedShell("cmake --build build --config Release", VulkanLoaderPath) + ( + $"cmake -S. -Bbuild -DUPDATE_DEPS=On -DCMAKE_BUILD_TYPE=Release{abi}{JobsArg}", + VulkanLoaderPath + ) + .AssertZeroExitCode(); + InheritedShell($"cmake --build build --config Release{JobsArg}", VulkanLoaderPath) .AssertZeroExitCode(); var runtimes = RootDirectory / "src" / "Native" / "Silk.NET.Vulkan.Loader.Native" / "runtimes"; if (OperatingSystem.IsWindows()) @@ -374,12 +381,82 @@ string AndroidHome ) ); + AbsolutePath AssimpPath => RootDirectory / "build" / "submodules" / "Assimp"; + + Target Assimp => CommonTarget + ( + x => x.Before(Compile) + .After(Clean) + .Executes + ( + () => + { + void CopyAs(AbsolutePath @out, string from, string to) + { + var file = @out.GlobFiles(from).First(); + CopyFile(file, to, FileExistsPolicy.Overwrite); + } + + var @out = AssimpPath / "build"; + var prepare = "cmake -S. -B build -D BUILD_SHARED_LIBS=ON"; + var build = $"cmake --build build --config Release{JobsArg}"; + EnsureCleanDirectory(@out); + var runtimes = RootDirectory / "src" / "Native" / "Silk.NET.Assimp.Native" / "runtimes"; + if (OperatingSystem.IsWindows()) + { + InheritedShell($"{prepare} -A X64", AssimpPath) + .AssertZeroExitCode(); + InheritedShell(build, AssimpPath) + .AssertZeroExitCode(); + + CopyAs(@out, "bin/Release/assimp-*-mt.dll", runtimes / "win-x64" / "native" / "Assimp64.dll"); + EnsureCleanDirectory(@out); + + InheritedShell($"{prepare} -A Win32", AssimpPath) + .AssertZeroExitCode(); + InheritedShell(build, AssimpPath) + .AssertZeroExitCode(); + + CopyAs(@out, "bin/Release/assimp-*-mt.dll", runtimes / "win-x86" / "native" / "Assimp32.dll"); + } + else if (OperatingSystem.IsLinux()) + { + InheritedShell($"{prepare} -DCMAKE_SYSTEM_PROCESSOR=x86_64", AssimpPath) + .AssertZeroExitCode(); + InheritedShell(build, AssimpPath) + .AssertZeroExitCode(); + + CopyAll(@out.GlobFiles("bin/libassimp.so.5"), runtimes / "linux-x64" / "native"); + } + else if (OperatingSystem.IsMacOS()) + { + InheritedShell($"{prepare} -DCMAKE_OSX_ARCHITECTURES=x86_64", AssimpPath) + .AssertZeroExitCode(); + InheritedShell(build, AssimpPath) + .AssertZeroExitCode(); + CopyAll(@out.GlobFiles("bin/libassimp.5.dylib"), runtimes / "osx-x64" / "native"); + + EnsureCleanDirectory(@out); + + InheritedShell($"{prepare} -DCMAKE_OSX_ARCHITECTURES=arm64", AssimpPath) + .AssertZeroExitCode(); + InheritedShell(build, AssimpPath) + .AssertZeroExitCode(); + + CopyAll(@out.GlobFiles("bin/libassimp.5.dylib"), runtimes / "osx-arm64" / "native"); + } + + PrUpdatedNativeBinary("Assimp"); + } + ) + ); + void PrUpdatedNativeBinary(string name) { var pushableToken = EnvironmentInfo.GetVariable("PUSHABLE_GITHUB_TOKEN"); var curBranch = GitCurrentBranch(RootDirectory); - if (GitHubActions.Instance?.GitHubRepository == "dotnet/Silk.NET" && - !string.IsNullOrWhiteSpace(pushableToken) && + if (!string.IsNullOrWhiteSpace(pushableToken) && + GitHubActions.Instance?.GitHubRepository == "dotnet/Silk.NET" && curBranch != "HEAD" && !string.IsNullOrWhiteSpace(curBranch) && !curBranch.StartsWith("ci/", StringComparison.OrdinalIgnoreCase) && // ignore other CI branches @@ -387,15 +464,30 @@ void PrUpdatedNativeBinary(string name) !curBranch.StartsWith("develop/", StringComparison.OrdinalIgnoreCase)) { // it's assumed that the pushable token was used to checkout the repo + var suffix = string.Empty; + if (OperatingSystem.IsWindows()) + { + suffix = "/**/*.dll"; + } + else if (OperatingSystem.IsMacOS()) + { + suffix = "/**/*.dylib"; + } + else if (OperatingSystem.IsLinux()) + { + suffix = "/**/*.so*"; + } + Git("fetch --all", RootDirectory); Git("pull"); - Git("add src/Native", RootDirectory); + Git($"add src/Native{suffix}", RootDirectory); var newBranch = $"ci/{curBranch}/{name.ToLower().Replace(' ', '_')}_bins"; var curCommit = GitCurrentCommit(RootDirectory); var commitCmd = InheritedShell - ( - $"git commit -m \"New binaries for {name} on {RuntimeInformation.OSDescription}\"" - ).AssertWaitForExit(); + ( + $"git commit -m \"New binaries for {name} on {RuntimeInformation.OSDescription}\"" + ) + .AssertWaitForExit(); if (!commitCmd.Output.Any(x => x.Text.Contains("nothing to commit", StringComparison.OrdinalIgnoreCase))) { commitCmd.AssertZeroExitCode(); diff --git a/src/infrastructure/Silk.NET.NUKE/Build.Packaging.cs b/src/infrastructure/Silk.NET.NUKE/Build.Packaging.cs index 3e8edfd163..24c28e46fc 100644 --- a/src/infrastructure/Silk.NET.NUKE/Build.Packaging.cs +++ b/src/infrastructure/Silk.NET.NUKE/Build.Packaging.cs @@ -15,7 +15,7 @@ partial class Build .Produces("build/output_packages/*.nupkg") .Executes ( - () => DotNetPack + () => ErrorsOnly ( s => s.SetProject(Solution) .SetConfiguration(Configuration) diff --git a/src/infrastructure/Silk.NET.NUKE/Build.ReviewHelpers.cs b/src/infrastructure/Silk.NET.NUKE/Build.ReviewHelpers.cs index c6ef015d3e..e9cea84578 100644 --- a/src/infrastructure/Silk.NET.NUKE/Build.ReviewHelpers.cs +++ b/src/infrastructure/Silk.NET.NUKE/Build.ReviewHelpers.cs @@ -20,8 +20,8 @@ partial class Build ( () => { - var files = RootDirectory.GlobFiles("**/*.csproj").ToArray(); - Logger.Info($"Found {files.Length} csproj files in \"{RootDirectory}\""); + var files = SourceDirectory.GlobFiles("**/*.csproj").ToArray(); + Logger.Info($"Found {files.Length} csproj files in \"{SourceDirectory}\""); var missedOut = new List(); foreach (var file in files) { @@ -44,7 +44,7 @@ partial class Build "extension) to the AllowedExclusions array in the NUKE Build.CI.AutoReview.cs file." ); - missedOut.Add(Path.GetRelativePath(RootDirectory, file)); + missedOut.Add(Path.GetRelativePath(RootDirectory, file).Replace('\\', '/')); } } diff --git a/src/infrastructure/Silk.NET.NUKE/Build.Support.cs b/src/infrastructure/Silk.NET.NUKE/Build.Support.cs index 003d4a2470..f81eddbd49 100644 --- a/src/infrastructure/Silk.NET.NUKE/Build.Support.cs +++ b/src/infrastructure/Silk.NET.NUKE/Build.Support.cs @@ -16,11 +16,12 @@ using Nuke.Common.IO; using Nuke.Common.ProjectModel; using Nuke.Common.Tooling; +using Nuke.Common.Tools.DotNet; using Nuke.Common.Utilities; using Octokit; -using Octokit.Internal; using static Nuke.Common.IO.FileSystemTasks; using static Nuke.Common.Tooling.ProcessTasks; +using static Nuke.Common.Tools.DotNet.DotNetTasks; partial class Build { @@ -32,6 +33,10 @@ partial class Build /// - Microsoft VisualStudio https://nuke.build/visualstudio /// - Microsoft VSCode https://nuke.build/vscode public static int Main() => Execute(x => x.Compile); + + [Parameter("Outputs build warnings instead of keeping the MSBuild logging quiet with just errors.")] + bool Warnings; + static int IndexOfOrThrow(string x, char y) { var idx = x.IndexOf(y); @@ -121,7 +126,7 @@ Target CommonTarget([CanBeNull] Target actualTarget = null) => Targets.GetOrAdd ); async Task AddOrUpdatePrComment(string type, string file, bool editOnly = false, params KeyValuePair[] subs) - {; + { var githubToken = EnvironmentInfo.GetVariable("GITHUB_TOKEN"); if (string.IsNullOrWhiteSpace(githubToken)) { @@ -152,7 +157,7 @@ async Task AddOrUpdatePrComment(string type, string file, bool editOnly = false, var github = new GitHubClient ( new ProductHeaderValue("Silk.NET-CI"), - new InMemoryCredentialStore(new Credentials(githubToken)) + new Octokit.Internal.InMemoryCredentialStore(new Credentials(githubToken)) ); var existingComment = (await github.Issue.Comment.GetAllForIssue("dotnet", "Silk.NET", pr)) @@ -185,4 +190,34 @@ async Task AddOrUpdatePrComment(string type, string file, bool editOnly = false, await github.Issue.Comment.Create("dotnet", "Silk.NET", pr, commentText); } } + + IReadOnlyCollection ErrorsOnly(Configure settings) where T : ToolSettings, new() + { + var toolSettings = settings(new T()); + var arguments = toolSettings.GetProcessArguments(); + + var finalArgs = arguments.RenderForExecution(); + if (!Warnings) + { + finalArgs += " /clp:ErrorsOnly"; + } + + using var proc = StartProcess + ( + toolSettings.ProcessToolPath, + finalArgs, + toolSettings.ProcessWorkingDirectory, + toolSettings.ProcessEnvironmentVariables, + toolSettings.ProcessExecutionTimeout, + toolSettings.ProcessLogOutput, + toolSettings.ProcessLogInvocation, + toolSettings.ProcessLogTimestamp, + toolSettings.ProcessLogFile, + toolSettings.ProcessCustomLogger, + arguments.FilterSecrets + ); + + proc.AssertZeroExitCode(); + return proc.Output; + } }