Skip to content

Commit

Permalink
Add additional filter capabilities to dotnet-pgo tool. (#89853)
Browse files Browse the repository at this point in the history 
Mono adopted PGO in .net7 as a replacement for a
solution called profiled AOT in mono/mono used by Android
SDK. In the replaced solution there was a concept of
a stop trigger, meaning that the user could collect methods
up to a stop trigger (a method name) meaning that the
profiled AOT image would only include methods up to that point.

When using EventPipe and nettrace there is limited ability to
get the same fine grained control over what methods that ends
up in the nettrace file. dotnet-monitor includes ways to stop
tracing if it hits for example a specific method, but due to the
nature of EventPipe, there could still be additional methods added
to the trace when closing session. dotnet-trace currently don't offer
any ability to do something similar, but if implemented, it
would probably come with the same limitations as dotnet-monitor.

Adding better filter capabilities to dotnet-pgo would add additional
capabilities, giving users more control on what methods that gets
included into the generated mibc file. That would give Mono's
profiled AOT better control to include methods up to a stop trigger.

dotnet-pgo alread had capabilities to include events based on timestamp
interval. This commit extends that to select the lower/upper timestamp
based on a regular expression matching methods. This commit also
adds capabilities to add a method include/exclude filters using regular
expression, giving users fine grained control on what methods to
include/exclude in the generated mibc file.

The additional filter capabilities could be used by Android SDK to
for example create a mibc file including all methods up to Main,
replacing the stop trigger features used in old profiled AOT solution.

---------

Co-authored-by: mdh1418 <mitchhwang1418@gmail.com>
  • Loading branch information
@lateralusX @mdh1418
lateralusX and mdh1418 authored Aug 10, 2023
1 parent 4aea2aa commit 6d9baea
Show file tree
Hide file tree
Showing 2 changed files with 127 additions and 2 deletions.
12 changes: 12 additions & 0 deletions src/coreclr/tools/dotnet-pgo/PgoRootCommand.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,14 @@ internal sealed class PgoRootCommand : CliRootCommand
new("--exclude-events-before") { DefaultValueFactory = _ => Double.MinValue, Description = "Exclude data from events before specified time. Time is specified as milliseconds from the start of the trace" };
public CliOption<double> ExcludeEventsAfter { get; } =
new("--exclude-events-after") { DefaultValueFactory = _ => Double.MaxValue, Description = "Exclude data from events after specified time. Time is specified as milliseconds from the start of the trace" };
public CliOption<string> ExcludeEventsBeforeJittingMethod { get; } =
new("--exclude-events-before-jitting-method") { DefaultValueFactory = _ => string.Empty, Description = "Exclude data from events before observing a specific method getting jitted. Method is matched using a regular expression against the method name. Note that the method name is formatted the same as in PerfView which includes typed parameters." };
public CliOption<string> ExcludeEventsAfterJittingMethod { get; } =
new("--exclude-events-after-jitting-method") { DefaultValueFactory = _ => string.Empty, Description = "Exclude data from events after observing a specific method getting jitted. Method is matched using a regular expression against the method name. Note that the method name is formatted the same as in PerfView which includes typed parameters." };
public CliOption<string> IncludeMethods { get; } =
new("--include-methods") { DefaultValueFactory = _ => string.Empty, Description = "Include methods with names matching regular expression. Note that the method names are formatted the same as in PerfView which includes typed parameters." };
public CliOption<string> ExcludeMethods { get; } =
new("--exclude-methods") { DefaultValueFactory = _ => string.Empty, Description = "Exclude methods with names matching regular expression. Note that the method names are formatted the same as in PerfView which includes typed parameters." };
public CliOption<bool> Compressed { get; } =
new("--compressed") { DefaultValueFactory = _ => true, Description = "Generate compressed mibc" };
public CliOption<int> DumpWorstOverlapGraphs { get; } =
Expand Down Expand Up @@ -99,6 +107,10 @@ public PgoRootCommand(string[] args) : base(".NET PGO Tool")
ClrInstanceId,
ExcludeEventsBefore,
ExcludeEventsAfter,
ExcludeEventsBeforeJittingMethod,
ExcludeEventsAfterJittingMethod,
IncludeMethods,
ExcludeMethods,
AutomaticReferences,
_verbosity,
Compressed,
Expand Down
117 changes: 115 additions & 2 deletions src/coreclr/tools/dotnet-pgo/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
using System.Text;
using System.Text.Json;
using System.Text.Encodings.Web;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

using Microsoft.Diagnostics.Tools.Pgo;
Expand Down Expand Up @@ -1343,6 +1344,12 @@ private int InnerProcessTraceFileMain()

double excludeEventsBefore = Get(_command.ExcludeEventsBefore);
double excludeEventsAfter = Get(_command.ExcludeEventsAfter);
Regex excludeEventsBeforeJittingMethod = !string.IsNullOrEmpty(Get(_command.ExcludeEventsBeforeJittingMethod)) ? new Regex(Get(_command.ExcludeEventsBeforeJittingMethod)) : null;
Regex excludeEventsAfterJittingMethod = !string.IsNullOrEmpty(Get(_command.ExcludeEventsAfterJittingMethod)) ? new Regex(Get(_command.ExcludeEventsAfterJittingMethod)) : null;
Regex includeMethods = !string.IsNullOrEmpty(Get(_command.IncludeMethods)) ? new Regex(Get(_command.IncludeMethods)) : null;
Regex excludeMethods = !string.IsNullOrEmpty(Get(_command.ExcludeMethods)) ? new Regex(Get(_command.ExcludeMethods)) : null;

// Find all the R2RLoad events.
if (_command.ProcessR2REvents)
{
foreach (var e in p.EventsInProcess.ByEventType<R2RGetEntryPointTraceData>())
Expand All @@ -1351,6 +1358,7 @@ private int InnerProcessTraceFileMain()
string retArg = e.MethodSignature.Substring(0, parenIndex);
string paramsArgs = e.MethodSignature.Substring(parenIndex);
string methodNameFromEventDirectly = retArg + e.MethodNamespace + "." + e.MethodName + paramsArgs;

if (e.ClrInstanceID != clrInstanceId)
{
if (!_command.Warnings)
Expand All @@ -1359,6 +1367,7 @@ private int InnerProcessTraceFileMain()
PrintWarning($"Skipped R2REntryPoint {methodNameFromEventDirectly} due to ClrInstanceID of {e.ClrInstanceID}");
continue;
}

MethodDesc method = null;
string extraWarningText = null;
bool failedDueToNonloadableModule = false;
Expand All @@ -1382,8 +1391,80 @@ private int InnerProcessTraceFileMain()
continue;
}

if ((e.TimeStampRelativeMSec >= excludeEventsBefore) && (e.TimeStampRelativeMSec <= excludeEventsAfter))
if (e.TimeStampRelativeMSec < excludeEventsBefore)
{
continue;
}

if (e.TimeStampRelativeMSec > excludeEventsAfter)
{
break;
}

string perfviewMethodName = e.MethodNamespace + "." + e.MethodName + paramsArgs;
if (PassesMethodFilter(includeMethods, excludeMethods, perfviewMethodName))
{
methodsToAttemptToPrepare.Add((int)e.EventIndex, new ProcessedMethodData(e.TimeStampRelativeMSec, method, "R2RLoad"));
}
}
}

// In case requesting events before/after jitting a method, discover the
// corresponding excludeEventsBefore/excludeEventsAfter in event stream based
// on filter criterias.
if (_command.ProcessJitEvents && (excludeEventsBeforeJittingMethod != null || excludeEventsAfterJittingMethod != null))
{
double firstMatchEventsBeforeJittingMethod = double.PositiveInfinity;
double lastMatchEventsAfterJittingMethod = double.NegativeInfinity;
foreach (var e in p.EventsInProcess.ByEventType<MethodJittingStartedTraceData>())
{
if (e.ClrInstanceID != clrInstanceId)
{
continue;
}

MethodDesc method = null;
bool failedDueToNonloadableModule = false;
try
{
method = idParser.ResolveMethodID(e.MethodID, out failedDueToNonloadableModule, false);
}
catch { }

if (method == null)
{
continue;
}

int parenIndex = e.MethodSignature.IndexOf('(');
string paramsArgs = e.MethodSignature.Substring(parenIndex);
string perfviewMethodName = e.MethodNamespace + "." + e.MethodName + paramsArgs;

if (e.TimeStampRelativeMSec > excludeEventsBefore && e.TimeStampRelativeMSec < firstMatchEventsBeforeJittingMethod && excludeEventsBeforeJittingMethod != null && excludeEventsBeforeJittingMethod.IsMatch(perfviewMethodName))
{
firstMatchEventsBeforeJittingMethod = e.TimeStampRelativeMSec;
}

if (e.TimeStampRelativeMSec < excludeEventsAfter && e.TimeStampRelativeMSec > lastMatchEventsAfterJittingMethod && excludeEventsAfterJittingMethod != null && excludeEventsAfterJittingMethod.IsMatch(perfviewMethodName))
{
lastMatchEventsAfterJittingMethod = e.TimeStampRelativeMSec;
}
}

if (firstMatchEventsBeforeJittingMethod < double.PositiveInfinity)
{
excludeEventsBefore = firstMatchEventsBeforeJittingMethod;
}

if (lastMatchEventsAfterJittingMethod > double.NegativeInfinity)
{
excludeEventsAfter = lastMatchEventsAfterJittingMethod;
}

if (excludeEventsBefore > excludeEventsAfter)
{
PrintError($"Exclude events before timestamp: \"{excludeEventsBefore}\" can't be later than exclude events after timestamp: \"{excludeEventsAfter}\"");
return -1;
}
}

Expand All @@ -1396,6 +1477,7 @@ private int InnerProcessTraceFileMain()
string retArg = e.MethodSignature.Substring(0, parenIndex);
string paramsArgs = e.MethodSignature.Substring(parenIndex);
string methodNameFromEventDirectly = retArg + e.MethodNamespace + "." + e.MethodName + paramsArgs;

if (e.ClrInstanceID != clrInstanceId)
{
if (!_command.Warnings)
Expand Down Expand Up @@ -1428,8 +1510,21 @@ private int InnerProcessTraceFileMain()
continue;
}

if ((e.TimeStampRelativeMSec >= excludeEventsBefore) && (e.TimeStampRelativeMSec <= excludeEventsAfter))
if (e.TimeStampRelativeMSec < excludeEventsBefore)
{
continue;
}

if (e.TimeStampRelativeMSec > excludeEventsAfter)
{
break;
}

string perfviewMethodName = e.MethodNamespace + "." + e.MethodName + paramsArgs;
if (PassesMethodFilter(includeMethods, excludeMethods, perfviewMethodName))
{
methodsToAttemptToPrepare.Add((int)e.EventIndex, new ProcessedMethodData(e.TimeStampRelativeMSec, method, "JitStart"));
}
}
}

Expand Down Expand Up @@ -1783,6 +1878,24 @@ void AddToInstrumentationData(int eventClrInstanceId, long methodID, int methodF
return 0;
}

private static bool PassesMethodFilter(Regex includeMethods, Regex excludeMethods, string methodName)
{
if (includeMethods != null || excludeMethods != null)
{
if (includeMethods != null && !includeMethods.IsMatch(methodName))
{
return false;
}

if (excludeMethods != null && excludeMethods.IsMatch(methodName))
{
return false;
}
}

return true;
}

private static void GenerateJittraceFile(FileInfo outputFileName, IEnumerable<ProcessedMethodData> methodsToAttemptToPrepare, JitTraceOptions jittraceOptions)
{
PrintMessage($"JitTrace options {jittraceOptions}");
Expand Down

0 comments on commit 6d9baea

Please sign in to comment.