Skip to content

Commit

Permalink
[acceptance-tests] Some improvements to the profiler stress runner.
Browse files Browse the repository at this point in the history 
* Print combined stdout and stderr in console output.
* Print the RNG seed used for a test run on startup and in the test report.
* Set the individual test timeout back to 6 hours.
* Use the nodefaults profiler option.
* Allow configuration through environment variables:
 - MONO_PROFILER_STRESS_OPTIONS: Specify a set of profiler options to use
   instead of the randomly generated ones.
 - MONO_PROFILER_STRESS_REGEX: Specify a regular expression to filter which
   tests should be run.
 - MONO_PROFILER_STRESS_SEED: Specify the RNG seed. Useful for reproducing full
   test runs at a later point.
 - MONO_PROFILER_STRESS_TIMEOUT: Specify a different individual test timeout
   from the default.
  • Loading branch information
Alex Rønne Petersen committed Jan 30, 2018
1 parent 78f653d commit 1216780
Showing 1 changed file with 59 additions and 25 deletions.
84 changes: 59 additions & 25 deletions acceptance-tests/profiler-stress/runner.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ sealed class TestResult {
public int? ExitCode { get; set; }
public string StandardOutput { get; set; }
public string StandardError { get; set; }
public string CombinedOutput { get; set; }
}

static class Program {
Expand All @@ -57,7 +58,9 @@ static class Program {
"jit",
};

static readonly TimeSpan _timeout = TimeSpan.FromHours (9);
static readonly TimeSpan _timeout = TimeSpan.FromHours (6);

const RegexOptions _regexOptions = RegexOptions.Compiled | RegexOptions.Singleline;

static readonly Dictionary<string, Predicate<Benchmark>> _filters = new Dictionary<string, Predicate<Benchmark>> {
{ "ironjs-v8", FilterNotOnArm },
Expand Down Expand Up @@ -89,26 +92,50 @@ static string ReplaceInvalidXmlChars (string text) {

static int Main ()
{
var regex = new Regex (".*", _regexOptions);

if (Environment.GetEnvironmentVariable ("MONO_PROFILER_STRESS_REGEX") is string envRegex)
regex = new Regex (envRegex, _regexOptions);

var depDir = Path.Combine ("..", "external", "benchmarker");
var benchDir = Path.Combine (depDir, "benchmarks");
var testDir = Path.Combine (depDir, "tests");

var benchmarks = Directory.EnumerateFiles (benchDir, "*.benchmark")
.Select (Benchmark.Load)
.Where (b => !b.OnlyExplicit && b.ClientCommandLine == null && IsSupported (b))
.Where (b => !b.OnlyExplicit && b.ClientCommandLine == null && IsSupported (b) && regex.IsMatch (b.Name))
.OrderBy (b => b.Name)
.ToArray ();

var monoPath = Path.GetFullPath (Path.Combine ("..", "..", "runtime", "mono-wrapper"));
var classDir = Path.GetFullPath (Path.Combine ("..", "..", "mcs", "class", "lib", "net_4_x"));

var rand = new Random ();
var cpus = Environment.ProcessorCount;
var seed = Environment.TickCount;

var results = new List<TestResult> (benchmarks.Length);
if (Environment.GetEnvironmentVariable ("MONO_PROFILER_STRESS_SEED") is string envSeed)
seed = int.Parse (envSeed);

var timeout = (int) _timeout.TotalMilliseconds;

if (Environment.GetEnvironmentVariable ("MONO_PROFILER_STRESS_TIMEOUT") is string envTimeout)
timeout = (int) TimeSpan.Parse (envTimeout).TotalMilliseconds;

string options = null;

if (Environment.GetEnvironmentVariable ("MONO_PROFILER_STRESS_OPTIONS") is string envOptions)
options = envOptions;

var rand = new Random (seed);
var cpus = Environment.ProcessorCount;

var sw = Stopwatch.StartNew ();

Console.ForegroundColor = ConsoleColor.Magenta;
Console.WriteLine ($"[{sw.Elapsed.ToString ("G")}] Starting test session with seed: {seed}");
Console.ResetColor ();

var results = new List<TestResult> (benchmarks.Length);

for (var i = 0; i < benchmarks.Length; i++) {
var bench = benchmarks [i];

Expand All @@ -117,17 +144,24 @@ static int Main ()
var maxSamples = rand.Next (0, cpus * 2000 + 1);
var heapShotFreq = rand.Next (-10, 11);
var maxFrames = rand.Next (0, 33);
var options = _options.ToDictionary (x => x, _ => rand.Next (0, 2) == 1)
.Select (x => (x.Value ? string.Empty : "no") + x.Key)
.ToArray ();
var flags = _options.ToDictionary (x => x, _ => rand.Next (0, 2) == 1)
.Select (x => (x.Value ? string.Empty : "no") + x.Key)
.ToArray ();

var profOptions = "nodefaults,output=/dev/null,";

var profOptions = $"maxframes={maxFrames},{string.Join (",", options)},output=/dev/null";
if (options == null) {
profOptions += $"maxframes={maxFrames},";

if (sampleFreq > 0)
profOptions += $",sample{sampleMode}={sampleFreq},maxsamples={maxSamples}";
if (sampleFreq > 0)
profOptions += $"sample{sampleMode}={sampleFreq},maxsamples={maxSamples},";

if (heapShotFreq > 0)
profOptions += $",heapshot={heapShotFreq}gc";
if (heapShotFreq > 0)
profOptions += $"heapshot={heapShotFreq}gc,";

profOptions += string.Join (",", flags);
} else
profOptions += options;

var info = new ProcessStartInfo {
UseShellExecute = false,
Expand Down Expand Up @@ -157,17 +191,22 @@ static int Main ()

var stdout = new StringBuilder ();
var stderr = new StringBuilder ();
var combined = new StringBuilder ();

proc.OutputDataReceived += (sender, args) => {
if (args.Data != null)
lock (result)
lock (result) {
stdout.AppendLine (args.Data);
combined.AppendLine (args.Data);
}
};

proc.ErrorDataReceived += (sender, args) => {
if (args.Data != null)
lock (result)
lock (result) {
stderr.AppendLine (args.Data);
combined.AppendLine (args.Data);
}
};

result.Stopwatch.Start ();
Expand All @@ -177,7 +216,7 @@ static int Main ()
proc.BeginOutputReadLine ();
proc.BeginErrorReadLine ();

if (!proc.WaitForExit ((int) _timeout.TotalMilliseconds)) {
if (!proc.WaitForExit (timeout)) {
// Force a thread dump.
Syscall.kill (proc.Id, Signum.SIGQUIT);
Thread.Sleep (1000);
Expand All @@ -194,6 +233,7 @@ static int Main ()
lock (result) {
result.StandardOutput = stdout.ToString ();
result.StandardError = stderr.ToString ();
result.CombinedOutput = combined.ToString ();
}
}

Expand All @@ -205,16 +245,10 @@ static int Main ()

if (result.ExitCode != 0) {
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine ("===== stdout =====");
Console.ResetColor ();

Console.WriteLine (result.StandardOutput);

Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine ("===== stderr =====");
Console.WriteLine ("===== stdout + stderr =====");
Console.ResetColor ();

Console.WriteLine (result.StandardError);
Console.WriteLine (result.CombinedOutput);
}

results.Add (result);
Expand All @@ -233,7 +267,7 @@ static int Main ()

using (var writer = XmlWriter.Create ("TestResult-profiler-stress.xml", settings)) {
writer.WriteStartDocument ();
writer.WriteComment ("This file represents the results of running a test suite");
writer.WriteComment ($"This file represents the results of running a test suite (seed: {seed})");

writer.WriteStartElement ("test-results");
writer.WriteAttributeString ("name", "profiler-stress-tests.dummy");
Expand Down

0 comments on commit 1216780

Please sign in to comment.