From 6d9baea7c228dfbb11833ae08137bba6a0d262ac Mon Sep 17 00:00:00 2001 From: Johan Lorensson Date: Thu, 10 Aug 2023 13:06:11 +0200 Subject: [PATCH] Add additional filter capabilities to dotnet-pgo tool. (#89853) Mono adopted PGO in .net7 as a replacement for a solution called profiled AOT in mono/mono used by Android SDK. In the replaced solution there was a concept of a stop trigger, meaning that the user could collect methods up to a stop trigger (a method name) meaning that the profiled AOT image would only include methods up to that point. When using EventPipe and nettrace there is limited ability to get the same fine grained control over what methods that ends up in the nettrace file. dotnet-monitor includes ways to stop tracing if it hits for example a specific method, but due to the nature of EventPipe, there could still be additional methods added to the trace when closing session. dotnet-trace currently don't offer any ability to do something similar, but if implemented, it would probably come with the same limitations as dotnet-monitor. Adding better filter capabilities to dotnet-pgo would add additional capabilities, giving users more control on what methods that gets included into the generated mibc file. That would give Mono's profiled AOT better control to include methods up to a stop trigger. dotnet-pgo alread had capabilities to include events based on timestamp interval. This commit extends that to select the lower/upper timestamp based on a regular expression matching methods. This commit also adds capabilities to add a method include/exclude filters using regular expression, giving users fine grained control on what methods to include/exclude in the generated mibc file. The additional filter capabilities could be used by Android SDK to for example create a mibc file including all methods up to Main, replacing the stop trigger features used in old profiled AOT solution. --------- Co-authored-by: mdh1418 --- .../tools/dotnet-pgo/PgoRootCommand.cs | 12 ++ src/coreclr/tools/dotnet-pgo/Program.cs | 117 +++++++++++++++++- 2 files changed, 127 insertions(+), 2 deletions(-) diff --git a/src/coreclr/tools/dotnet-pgo/PgoRootCommand.cs b/src/coreclr/tools/dotnet-pgo/PgoRootCommand.cs index 06f63db826e4e8..b9f3fab54043ce 100644 --- a/src/coreclr/tools/dotnet-pgo/PgoRootCommand.cs +++ b/src/coreclr/tools/dotnet-pgo/PgoRootCommand.cs @@ -45,6 +45,14 @@ internal sealed class PgoRootCommand : CliRootCommand new("--exclude-events-before") { DefaultValueFactory = _ => Double.MinValue, Description = "Exclude data from events before specified time. Time is specified as milliseconds from the start of the trace" }; public CliOption ExcludeEventsAfter { get; } = new("--exclude-events-after") { DefaultValueFactory = _ => Double.MaxValue, Description = "Exclude data from events after specified time. Time is specified as milliseconds from the start of the trace" }; + public CliOption ExcludeEventsBeforeJittingMethod { get; } = + new("--exclude-events-before-jitting-method") { DefaultValueFactory = _ => string.Empty, Description = "Exclude data from events before observing a specific method getting jitted. Method is matched using a regular expression against the method name. Note that the method name is formatted the same as in PerfView which includes typed parameters." }; + public CliOption ExcludeEventsAfterJittingMethod { get; } = + new("--exclude-events-after-jitting-method") { DefaultValueFactory = _ => string.Empty, Description = "Exclude data from events after observing a specific method getting jitted. Method is matched using a regular expression against the method name. Note that the method name is formatted the same as in PerfView which includes typed parameters." }; + public CliOption IncludeMethods { get; } = + new("--include-methods") { DefaultValueFactory = _ => string.Empty, Description = "Include methods with names matching regular expression. Note that the method names are formatted the same as in PerfView which includes typed parameters." }; + public CliOption ExcludeMethods { get; } = + new("--exclude-methods") { DefaultValueFactory = _ => string.Empty, Description = "Exclude methods with names matching regular expression. Note that the method names are formatted the same as in PerfView which includes typed parameters." }; public CliOption Compressed { get; } = new("--compressed") { DefaultValueFactory = _ => true, Description = "Generate compressed mibc" }; public CliOption DumpWorstOverlapGraphs { get; } = @@ -99,6 +107,10 @@ public PgoRootCommand(string[] args) : base(".NET PGO Tool") ClrInstanceId, ExcludeEventsBefore, ExcludeEventsAfter, + ExcludeEventsBeforeJittingMethod, + ExcludeEventsAfterJittingMethod, + IncludeMethods, + ExcludeMethods, AutomaticReferences, _verbosity, Compressed, diff --git a/src/coreclr/tools/dotnet-pgo/Program.cs b/src/coreclr/tools/dotnet-pgo/Program.cs index c65c1d22c86a86..400d03ccf321fd 100644 --- a/src/coreclr/tools/dotnet-pgo/Program.cs +++ b/src/coreclr/tools/dotnet-pgo/Program.cs @@ -21,6 +21,7 @@ using System.Text; using System.Text.Json; using System.Text.Encodings.Web; +using System.Text.RegularExpressions; using System.Threading.Tasks; using Microsoft.Diagnostics.Tools.Pgo; @@ -1343,6 +1344,12 @@ private int InnerProcessTraceFileMain() double excludeEventsBefore = Get(_command.ExcludeEventsBefore); double excludeEventsAfter = Get(_command.ExcludeEventsAfter); + Regex excludeEventsBeforeJittingMethod = !string.IsNullOrEmpty(Get(_command.ExcludeEventsBeforeJittingMethod)) ? new Regex(Get(_command.ExcludeEventsBeforeJittingMethod)) : null; + Regex excludeEventsAfterJittingMethod = !string.IsNullOrEmpty(Get(_command.ExcludeEventsAfterJittingMethod)) ? new Regex(Get(_command.ExcludeEventsAfterJittingMethod)) : null; + Regex includeMethods = !string.IsNullOrEmpty(Get(_command.IncludeMethods)) ? new Regex(Get(_command.IncludeMethods)) : null; + Regex excludeMethods = !string.IsNullOrEmpty(Get(_command.ExcludeMethods)) ? new Regex(Get(_command.ExcludeMethods)) : null; + + // Find all the R2RLoad events. if (_command.ProcessR2REvents) { foreach (var e in p.EventsInProcess.ByEventType()) @@ -1351,6 +1358,7 @@ private int InnerProcessTraceFileMain() string retArg = e.MethodSignature.Substring(0, parenIndex); string paramsArgs = e.MethodSignature.Substring(parenIndex); string methodNameFromEventDirectly = retArg + e.MethodNamespace + "." + e.MethodName + paramsArgs; + if (e.ClrInstanceID != clrInstanceId) { if (!_command.Warnings) @@ -1359,6 +1367,7 @@ private int InnerProcessTraceFileMain() PrintWarning($"Skipped R2REntryPoint {methodNameFromEventDirectly} due to ClrInstanceID of {e.ClrInstanceID}"); continue; } + MethodDesc method = null; string extraWarningText = null; bool failedDueToNonloadableModule = false; @@ -1382,8 +1391,80 @@ private int InnerProcessTraceFileMain() continue; } - if ((e.TimeStampRelativeMSec >= excludeEventsBefore) && (e.TimeStampRelativeMSec <= excludeEventsAfter)) + if (e.TimeStampRelativeMSec < excludeEventsBefore) + { + continue; + } + + if (e.TimeStampRelativeMSec > excludeEventsAfter) + { + break; + } + + string perfviewMethodName = e.MethodNamespace + "." + e.MethodName + paramsArgs; + if (PassesMethodFilter(includeMethods, excludeMethods, perfviewMethodName)) + { methodsToAttemptToPrepare.Add((int)e.EventIndex, new ProcessedMethodData(e.TimeStampRelativeMSec, method, "R2RLoad")); + } + } + } + + // In case requesting events before/after jitting a method, discover the + // corresponding excludeEventsBefore/excludeEventsAfter in event stream based + // on filter criterias. + if (_command.ProcessJitEvents && (excludeEventsBeforeJittingMethod != null || excludeEventsAfterJittingMethod != null)) + { + double firstMatchEventsBeforeJittingMethod = double.PositiveInfinity; + double lastMatchEventsAfterJittingMethod = double.NegativeInfinity; + foreach (var e in p.EventsInProcess.ByEventType()) + { + if (e.ClrInstanceID != clrInstanceId) + { + continue; + } + + MethodDesc method = null; + bool failedDueToNonloadableModule = false; + try + { + method = idParser.ResolveMethodID(e.MethodID, out failedDueToNonloadableModule, false); + } + catch { } + + if (method == null) + { + continue; + } + + int parenIndex = e.MethodSignature.IndexOf('('); + string paramsArgs = e.MethodSignature.Substring(parenIndex); + string perfviewMethodName = e.MethodNamespace + "." + e.MethodName + paramsArgs; + + if (e.TimeStampRelativeMSec > excludeEventsBefore && e.TimeStampRelativeMSec < firstMatchEventsBeforeJittingMethod && excludeEventsBeforeJittingMethod != null && excludeEventsBeforeJittingMethod.IsMatch(perfviewMethodName)) + { + firstMatchEventsBeforeJittingMethod = e.TimeStampRelativeMSec; + } + + if (e.TimeStampRelativeMSec < excludeEventsAfter && e.TimeStampRelativeMSec > lastMatchEventsAfterJittingMethod && excludeEventsAfterJittingMethod != null && excludeEventsAfterJittingMethod.IsMatch(perfviewMethodName)) + { + lastMatchEventsAfterJittingMethod = e.TimeStampRelativeMSec; + } + } + + if (firstMatchEventsBeforeJittingMethod < double.PositiveInfinity) + { + excludeEventsBefore = firstMatchEventsBeforeJittingMethod; + } + + if (lastMatchEventsAfterJittingMethod > double.NegativeInfinity) + { + excludeEventsAfter = lastMatchEventsAfterJittingMethod; + } + + if (excludeEventsBefore > excludeEventsAfter) + { + PrintError($"Exclude events before timestamp: \"{excludeEventsBefore}\" can't be later than exclude events after timestamp: \"{excludeEventsAfter}\""); + return -1; } } @@ -1396,6 +1477,7 @@ private int InnerProcessTraceFileMain() string retArg = e.MethodSignature.Substring(0, parenIndex); string paramsArgs = e.MethodSignature.Substring(parenIndex); string methodNameFromEventDirectly = retArg + e.MethodNamespace + "." + e.MethodName + paramsArgs; + if (e.ClrInstanceID != clrInstanceId) { if (!_command.Warnings) @@ -1428,8 +1510,21 @@ private int InnerProcessTraceFileMain() continue; } - if ((e.TimeStampRelativeMSec >= excludeEventsBefore) && (e.TimeStampRelativeMSec <= excludeEventsAfter)) + if (e.TimeStampRelativeMSec < excludeEventsBefore) + { + continue; + } + + if (e.TimeStampRelativeMSec > excludeEventsAfter) + { + break; + } + + string perfviewMethodName = e.MethodNamespace + "." + e.MethodName + paramsArgs; + if (PassesMethodFilter(includeMethods, excludeMethods, perfviewMethodName)) + { methodsToAttemptToPrepare.Add((int)e.EventIndex, new ProcessedMethodData(e.TimeStampRelativeMSec, method, "JitStart")); + } } } @@ -1783,6 +1878,24 @@ void AddToInstrumentationData(int eventClrInstanceId, long methodID, int methodF return 0; } + private static bool PassesMethodFilter(Regex includeMethods, Regex excludeMethods, string methodName) + { + if (includeMethods != null || excludeMethods != null) + { + if (includeMethods != null && !includeMethods.IsMatch(methodName)) + { + return false; + } + + if (excludeMethods != null && excludeMethods.IsMatch(methodName)) + { + return false; + } + } + + return true; + } + private static void GenerateJittraceFile(FileInfo outputFileName, IEnumerable methodsToAttemptToPrepare, JitTraceOptions jittraceOptions) { PrintMessage($"JitTrace options {jittraceOptions}");