From 7f94aa593c0d579b897a51f6422e4c9dd05ff2b2 Mon Sep 17 00:00:00 2001 From: Andy Ayers Date: Thu, 8 Jun 2023 12:53:02 -0700 Subject: [PATCH 1/2] Revise how we set tiering and pgo options for spmi benchmark collections Don't set these in the BDN environment; set them via the BDN command line so they only impact the process being benchmarked. Fixes #86410 --- src/coreclr/scripts/superpmi_benchmarks.py | 41 +++++++++++++--------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/src/coreclr/scripts/superpmi_benchmarks.py b/src/coreclr/scripts/superpmi_benchmarks.py index 8b78a0099c4dd..892c774c95913 100644 --- a/src/coreclr/scripts/superpmi_benchmarks.py +++ b/src/coreclr/scripts/superpmi_benchmarks.py @@ -209,22 +209,29 @@ def build_and_run(coreclr_args, output_mch_name): if benchmark_binary.lower().startswith("microbenchmarks"): # Disable ReadyToRun so we always JIT R2R methods and collect them collection_command = f"{dotnet_exe} {benchmarks_dll} --filter \"*\" --corerun {os.path.join(core_root, corerun_exe)} --partition-count {partition_count} " \ - f"--partition-index {partition_index} --envVars DOTNET_JitName:{shim_name} " \ - " DOTNET_ZapDisable:1 DOTNET_ReadyToRun:0 " \ - "--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --logBuildOutput" + f"--partition-index {partition_index} --iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --logBuildOutput " \ + f"--envVars DOTNET_JitName:{shim_name} DOTNET_ZapDisable:1 DOTNET_ReadyToRun:0" + elif benchmark_binary.lower().startswith("demobenchmarks"): # Disable ReadyToRun so we always JIT R2R methods and collect them collection_command = f"{dotnet_exe} {benchmarks_dll} -f *CollisionBatcherTaskBenchmarks.* *GroupedCollisionTesterBenchmarks.* *GatherScatterBenchmarks.* " \ - " *OneBodyConstraintBenchmarks.* *TwoBodyConstraintBenchmarks.* *ThreeBodyConstraintBenchmarks.* *FourBodyConstraintBenchmarks.* " \ - " *SweepBenchmarks.* *ShapeRayBenchmarks.* *ShapePileBenchmark.* *RagdollTubeBenchmark.* " \ - f" --corerun {os.path.join(core_root, corerun_exe)} --envVars DOTNET_JitName:{shim_name} " \ - " DOTNET_ZapDisable:1 DOTNET_ReadyToRun:0 " \ - "--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --logBuildOutput" + " *OneBodyConstraintBenchmarks.* *TwoBodyConstraintBenchmarks.* *ThreeBodyConstraintBenchmarks.* *FourBodyConstraintBenchmarks.* " \ + " *SweepBenchmarks.* *ShapeRayBenchmarks.* *ShapePileBenchmark.* *RagdollTubeBenchmark.* " \ + f" --corerun {os.path.join(core_root, corerun_exe)} " \ + "--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --logBuildOutput " \ + f" --envVars DOTNET_JitName:{shim_name} DOTNET_ZapDisable:1 DOTNET_ReadyToRun:0" else: # Disable ReadyToRun so we always JIT R2R methods and collect them - collection_command = f"{dotnet_exe} {benchmarks_dll} --filter \"*\" --corerun {os.path.join(core_root, corerun_exe)} --envVars DOTNET_JitName:{shim_name} " \ - " DOTNET_ZapDisable:1 DOTNET_ReadyToRun:0 " \ - "--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --logBuildOutput" + collection_command = f"{dotnet_exe} {benchmarks_dll} --filter \"*\" --corerun {os.path.join(core_root, corerun_exe)} " \ + "--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --logBuildOutput " \ + f"--envVars DOTNET_JitName:{shim_name} DOTNET_ZapDisable:1 DOTNET_ReadyToRun:0" + + if coreclr_args.tiered_pgo: + collection_command += " DOTNET_TieredCompilation:1 DOTNET_TieredPGO:1" + elif coreclr_args.tiered_compilation: + collection_command += " DOTNET_TieredCompilation:1" + else: + collection_command += " DOTNET_TieredCompilation:0" # Generate the execution script in Temp location with TempDir() as temp_location: @@ -232,13 +239,18 @@ def build_and_run(coreclr_args, output_mch_name): contents = [] # Unset the JitName so dotnet process will not fail + # Unset TieredCompilation and TieredPGO so the parent BDN process is just running with defaults if is_windows: contents.append("set JitName=%DOTNET_JitName%") contents.append("set DOTNET_JitName=") + contents.append("set DOTNET_TieredCompilation=") + contents.append("set DOTNET_TieredPGO=") else: contents.append("#!/bin/bash") contents.append("export JitName=$DOTNET_JitName") contents.append("unset DOTNET_JitName") + contents.append("unset DOTNET_TieredCompilation") + contents.append("unset DOTNET_TieredPGO") contents.append(f"pushd {performance_directory}") contents.append(collection_command) @@ -262,12 +274,7 @@ def build_and_run(coreclr_args, output_mch_name): "-output_mch_path", output_mch_name, "-log_level", "debug"] - if coreclr_args.tiered_compilation: - script_args.append("--tiered_compilation"); - elif coreclr_args.tiered_pgo: - script_args.append("--tiered_pgo"); - - script_args.append(script_name); + script_args.append(script_name) run_command(script_args, _exit_on_fail=True) From ad2ee7fd54e0f54e8e284899ac900bbd993eca52 Mon Sep 17 00:00:00 2001 From: Andy Ayers Date: Thu, 8 Jun 2023 16:06:03 -0700 Subject: [PATCH 2/2] pull out common parts of BDN command line --- src/coreclr/scripts/superpmi_benchmarks.py | 37 +++++++++++----------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/src/coreclr/scripts/superpmi_benchmarks.py b/src/coreclr/scripts/superpmi_benchmarks.py index 892c774c95913..a20d39368df8a 100644 --- a/src/coreclr/scripts/superpmi_benchmarks.py +++ b/src/coreclr/scripts/superpmi_benchmarks.py @@ -206,32 +206,33 @@ def build_and_run(coreclr_args, output_mch_name): "--framework", "net8.0", "--no-restore", "/p:NuGetPackageRoot=" + artifacts_packages_directory, "-o", artifacts_directory], _exit_on_fail=True) - if benchmark_binary.lower().startswith("microbenchmarks"): - # Disable ReadyToRun so we always JIT R2R methods and collect them - collection_command = f"{dotnet_exe} {benchmarks_dll} --filter \"*\" --corerun {os.path.join(core_root, corerun_exe)} --partition-count {partition_count} " \ - f"--partition-index {partition_index} --iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --logBuildOutput " \ - f"--envVars DOTNET_JitName:{shim_name} DOTNET_ZapDisable:1 DOTNET_ReadyToRun:0" + # common BDN prefix + collection_command = f"{dotnet_exe} {benchmarks_dll} --corerun {os.path.join(core_root, corerun_exe)} " + # test specific filters + if benchmark_binary.lower().startswith("microbenchmarks"): + collection_command += f"--filter \"*\" --partition-count {partition_count} --partition-index {partition_index} " elif benchmark_binary.lower().startswith("demobenchmarks"): - # Disable ReadyToRun so we always JIT R2R methods and collect them - collection_command = f"{dotnet_exe} {benchmarks_dll} -f *CollisionBatcherTaskBenchmarks.* *GroupedCollisionTesterBenchmarks.* *GatherScatterBenchmarks.* " \ + collection_command += "-f *CollisionBatcherTaskBenchmarks.* *GroupedCollisionTesterBenchmarks.* *GatherScatterBenchmarks.* " \ " *OneBodyConstraintBenchmarks.* *TwoBodyConstraintBenchmarks.* *ThreeBodyConstraintBenchmarks.* *FourBodyConstraintBenchmarks.* " \ - " *SweepBenchmarks.* *ShapeRayBenchmarks.* *ShapePileBenchmark.* *RagdollTubeBenchmark.* " \ - f" --corerun {os.path.join(core_root, corerun_exe)} " \ - "--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --logBuildOutput " \ - f" --envVars DOTNET_JitName:{shim_name} DOTNET_ZapDisable:1 DOTNET_ReadyToRun:0" + " *SweepBenchmarks.* *ShapeRayBenchmarks.* *ShapePileBenchmark.* *RagdollTubeBenchmark.* " else: - # Disable ReadyToRun so we always JIT R2R methods and collect them - collection_command = f"{dotnet_exe} {benchmarks_dll} --filter \"*\" --corerun {os.path.join(core_root, corerun_exe)} " \ - "--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --logBuildOutput " \ - f"--envVars DOTNET_JitName:{shim_name} DOTNET_ZapDisable:1 DOTNET_ReadyToRun:0" + collection_command += "--filter \"*\" " + + # common BDN arguments + collection_command += "--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --logBuildOutput " + + # common BDN environment var settings + # Disable ReadyToRun so we always JIT R2R methods and collect them + collection_command += f"--envVars DOTNET_JitName:{shim_name} DOTNET_ZapDisable:1 DOTNET_ReadyToRun:0 " + # custom BDN environment var settings if coreclr_args.tiered_pgo: - collection_command += " DOTNET_TieredCompilation:1 DOTNET_TieredPGO:1" + collection_command += "DOTNET_TieredCompilation:1 DOTNET_TieredPGO:1" elif coreclr_args.tiered_compilation: - collection_command += " DOTNET_TieredCompilation:1" + collection_command += "DOTNET_TieredCompilation:1 DOTNET_TieredPGO:0" else: - collection_command += " DOTNET_TieredCompilation:0" + collection_command += "DOTNET_TieredCompilation:0" # Generate the execution script in Temp location with TempDir() as temp_location: