From a4b3772d1ada339220145843138d4c622134adeb Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 9 Jun 2021 17:42:14 -0700 Subject: [PATCH] Remove crossgen from superpmi tools (#53958) * Remove crossgen SPMI collection * Remove reference of crossgen from superpmi.py * some misc crossgen references * put back deleted things --- eng/pipelines/coreclr/superpmi.yml | 21 ---- src/coreclr/ToolBox/superpmi/readme.md | 2 +- .../superpmi-shim-collector/icorjitinfo.cpp | 2 +- .../ToolBox/superpmi/superpmi/icorjitinfo.cpp | 2 +- src/coreclr/scripts/superpmi.py | 106 ++---------------- 5 files changed, 11 insertions(+), 122 deletions(-) diff --git a/eng/pipelines/coreclr/superpmi.yml b/eng/pipelines/coreclr/superpmi.yml index 8219ad02b8387..d6325d87b74eb 100644 --- a/eng/pipelines/coreclr/superpmi.yml +++ b/eng/pipelines/coreclr/superpmi.yml @@ -77,27 +77,6 @@ jobs: collectionType: pmi collectionName: libraries -- template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/superpmi-job.yml - buildConfig: checked - platforms: - # Linux tests are built on the OSX machines. - # - OSX_x64 - - Linux_arm - - Linux_arm64 - - Linux_x64 - - windows_x64 - - windows_x86 - - windows_arm64 - helixQueueGroup: ci - helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml - jobParameters: - testGroup: outerloop - liveLibrariesBuildConfig: Release - collectionType: crossgen - collectionName: libraries - - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/coreclr/templates/superpmi-job.yml diff --git a/src/coreclr/ToolBox/superpmi/readme.md b/src/coreclr/ToolBox/superpmi/readme.md index d9aded28c479a..c3de3dfe94288 100644 --- a/src/coreclr/ToolBox/superpmi/readme.md +++ b/src/coreclr/ToolBox/superpmi/readme.md @@ -91,7 +91,7 @@ speed up the disk operations. These are the general steps that are followed when doing a SuperPMI collection: 1. Collect .MC files. Set up for collection, then cause the JIT to be invoked -by either running a scenario, running tests, crossgen compiling assemblies, +by either running a scenario, running tests, crossgen2 compiling assemblies, or using PMI to force the JIT to compile functions in an assembly. During collection, the data for each JIT compiled function is stored in a uniquely named file with a ".MC" filename extension (for "method context"). diff --git a/src/coreclr/ToolBox/superpmi/superpmi-shim-collector/icorjitinfo.cpp b/src/coreclr/ToolBox/superpmi/superpmi-shim-collector/icorjitinfo.cpp index cd601a2621333..de2c555c20a4c 100644 --- a/src/coreclr/ToolBox/superpmi/superpmi-shim-collector/icorjitinfo.cpp +++ b/src/coreclr/ToolBox/superpmi/superpmi-shim-collector/icorjitinfo.cpp @@ -2115,7 +2115,7 @@ uint16_t interceptor_ICJI::getRelocTypeHint(void* target) // For what machine does the VM expect the JIT to generate code? The VM // returns one of the IMAGE_FILE_MACHINE_* values. Note that if the VM -// is cross-compiling (such as the case for crossgen), it will return a +// is cross-compiling (such as the case for crossgen2), it will return a // different value than if it was compiling for the host architecture. // uint32_t interceptor_ICJI::getExpectedTargetArchitecture() diff --git a/src/coreclr/ToolBox/superpmi/superpmi/icorjitinfo.cpp b/src/coreclr/ToolBox/superpmi/superpmi/icorjitinfo.cpp index f1eb73b8915c9..91bf8bf2f779f 100644 --- a/src/coreclr/ToolBox/superpmi/superpmi/icorjitinfo.cpp +++ b/src/coreclr/ToolBox/superpmi/superpmi/icorjitinfo.cpp @@ -1859,7 +1859,7 @@ uint16_t MyICJI::getRelocTypeHint(void* target) // For what machine does the VM expect the JIT to generate code? The VM // returns one of the IMAGE_FILE_MACHINE_* values. Note that if the VM -// is cross-compiling (such as the case for crossgen), it will return a +// is cross-compiling (such as the case for crossgen2), it will return a // different value than if it was compiling for the host architecture. // uint32_t MyICJI::getExpectedTargetArchitecture() diff --git a/src/coreclr/scripts/superpmi.py b/src/coreclr/scripts/superpmi.py index 449c8d2dcb115..d4033201ca04e 100755 --- a/src/coreclr/scripts/superpmi.py +++ b/src/coreclr/scripts/superpmi.py @@ -262,9 +262,8 @@ collect_parser.add_argument("collection_args", nargs='?', help="Arguments to pass to the SuperPMI collect command. This is a single string; quote it if necessary if the arguments contain spaces.") collect_parser.add_argument("--pmi", action="store_true", help="Run PMI on a set of directories or assemblies.") -collect_parser.add_argument("--crossgen", action="store_true", help="Run crossgen on a set of directories or assemblies.") collect_parser.add_argument("--crossgen2", action="store_true", help="Run crossgen2 on a set of directories or assemblies.") -collect_parser.add_argument("-assemblies", dest="assemblies", nargs="+", default=[], help="A list of managed dlls or directories to recursively use while collecting with PMI, crossgen, or crossgen2. Required if --pmi, --crossgen, or --crossgen2 is specified.") +collect_parser.add_argument("-assemblies", dest="assemblies", nargs="+", default=[], help="A list of managed dlls or directories to recursively use while collecting with PMI or crossgen2. Required if --pmi or --crossgen2 is specified.") collect_parser.add_argument("-exclude", dest="exclude", nargs="+", default=[], help="A list of files or directories to exclude from the files and directories specified by `-assemblies`.") collect_parser.add_argument("-pmi_location", help="Path to pmi.dll to use during PMI run. Optional; pmi.dll will be downloaded from Azure Storage if necessary.") collect_parser.add_argument("-output_mch_path", help="Location to place the final MCH file.") @@ -906,15 +905,12 @@ def __init__(self, coreclr_args): if coreclr_args.host_os == "OSX": self.collection_shim_name = "libsuperpmi-shim-collector.dylib" self.corerun_tool_name = "corerun" - self.crossgen_tool_name = "crossgen" elif coreclr_args.host_os == "Linux": self.collection_shim_name = "libsuperpmi-shim-collector.so" self.corerun_tool_name = "corerun" - self.crossgen_tool_name = "crossgen" elif coreclr_args.host_os == "windows": self.collection_shim_name = "superpmi-shim-collector.dll" self.corerun_tool_name = "corerun.exe" - self.crossgen_tool_name = "crossgen.exe" else: raise RuntimeError("Unsupported OS.") @@ -931,9 +927,6 @@ def __init__(self, coreclr_args): self.pmi_location = determine_pmi_location(coreclr_args) self.corerun = os.path.join(self.core_root, self.corerun_tool_name) - if coreclr_args.crossgen: - self.crossgen_tool = os.path.join(self.core_root, self.crossgen_tool_name) - if coreclr_args.crossgen2: self.corerun = os.path.join(self.core_root, self.corerun_tool_name) if coreclr_args.dotnet_tool_path is None: @@ -942,7 +935,7 @@ def __init__(self, coreclr_args): self.crossgen2_driver_tool = coreclr_args.dotnet_tool_path logging.debug("Using crossgen2 driver tool %s", self.crossgen2_driver_tool) - if coreclr_args.pmi or coreclr_args.crossgen or coreclr_args.crossgen2: + if coreclr_args.pmi or coreclr_args.crossgen2: self.assemblies = coreclr_args.assemblies self.exclude = coreclr_args.exclude @@ -1081,7 +1074,7 @@ def set_and_report_env(env, root_env, complus_env = None): # If we need them, collect all the assemblies we're going to use for the collection(s). # Remove the files matching the `-exclude` arguments (case-insensitive) from the list. - if self.coreclr_args.pmi or self.coreclr_args.crossgen or self.coreclr_args.crossgen2: + if self.coreclr_args.pmi or self.coreclr_args.crossgen2: assemblies = [] for item in self.assemblies: assemblies += get_files_from_path(item, match_func=lambda file: any(file.endswith(extension) for extension in [".dll", ".exe"]) and (self.exclude is None or not any(e.lower() in file.lower() for e in self.exclude))) @@ -1180,84 +1173,6 @@ async def run_pmi(print_prefix, assembly, self): os.environ.update(old_env) ################################################################################################ end of "self.coreclr_args.pmi is True" - ################################################################################################ Do collection using crossgen - if self.coreclr_args.crossgen is True: - logging.debug("Starting collection using crossgen") - - async def run_crossgen(print_prefix, assembly, self): - """ Run crossgen over all dlls - """ - - root_crossgen_output_filename = make_safe_filename("crossgen_" + assembly) + ".out.dll" - crossgen_output_assembly_filename = os.path.join(self.temp_location, root_crossgen_output_filename) - try: - if os.path.exists(crossgen_output_assembly_filename): - os.remove(crossgen_output_assembly_filename) - except OSError as ose: - if "[WinError 32] The process cannot access the file because it is being used by another " \ - "process:" in format(ose): - logging.warning("Skipping file %s. Got error: %s", crossgen_output_assembly_filename, ose) - return - else: - raise ose - - command = [self.crossgen_tool, "/Platform_Assemblies_Paths", self.core_root, "/in", assembly, "/out", crossgen_output_assembly_filename] - command_string = " ".join(command) - logging.debug("%s%s", print_prefix, command_string) - - # Save the stdout and stderr to files, so we can see if crossgen wrote any interesting messages. - # Use the name of the assembly as the basename of the file. mkstemp() will ensure the file - # is unique. - root_output_filename = make_safe_filename("crossgen_" + assembly + "_") - try: - stdout_file_handle, stdout_filepath = tempfile.mkstemp(suffix=".stdout", prefix=root_output_filename, dir=self.temp_location) - stderr_file_handle, stderr_filepath = tempfile.mkstemp(suffix=".stderr", prefix=root_output_filename, dir=self.temp_location) - - proc = await asyncio.create_subprocess_shell( - command_string, - stdout=stdout_file_handle, - stderr=stderr_file_handle) - - await proc.communicate() - - os.close(stdout_file_handle) - os.close(stderr_file_handle) - - # No need to keep zero-length files - if is_zero_length_file(stdout_filepath): - os.remove(stdout_filepath) - if is_zero_length_file(stderr_filepath): - os.remove(stderr_filepath) - - return_code = proc.returncode - if return_code != 0: - logging.debug("'%s': Error return code: %s", command_string, return_code) - write_file_to_log(stdout_filepath, log_level=logging.DEBUG) - - write_file_to_log(stderr_filepath, log_level=logging.DEBUG) - except OSError as ose: - if "[WinError 32] The process cannot access the file because it is being used by another " \ - "process:" in format(ose): - logging.warning("Skipping file %s. Got error: %s", root_output_filename, ose) - else: - raise ose - - # Set environment variables. - crossgen_command_env = env_copy.copy() - crossgen_complus_env = complus_env.copy() - crossgen_complus_env["JitName"] = self.collection_shim_name - set_and_report_env(crossgen_command_env, root_env, crossgen_complus_env) - - old_env = os.environ.copy() - os.environ.update(crossgen_command_env) - - helper = AsyncSubprocessHelper(assemblies, verbose=True) - helper.run_to_completion(run_crossgen, self) - - os.environ.clear() - os.environ.update(old_env) - ################################################################################################ end of "self.coreclr_args.crossgen is True" - ################################################################################################ Do collection using crossgen2 if self.coreclr_args.crossgen2 is True: logging.debug("Starting collection using crossgen2") @@ -3435,11 +3350,6 @@ def verify_replay_common_args(): lambda unused: True, "Unable to set pmi") - coreclr_args.verify(args, - "crossgen", - lambda unused: True, - "Unable to set crossgen") - coreclr_args.verify(args, "crossgen2", lambda unused: True, @@ -3512,8 +3422,8 @@ def verify_replay_common_args(): lambda unused: True, "Unable to set tiered_compilation") - if (args.collection_command is None) and (args.pmi is False) and (args.crossgen is False) and (args.crossgen2 is False): - print("Either a collection command or `--pmi` or `--crossgen` or `--crossgen2` must be specified") + if (args.collection_command is None) and (args.pmi is False) and (args.crossgen2 is False): + print("Either a collection command or `--pmi` or `--crossgen2` must be specified") sys.exit(1) if (args.collection_command is not None) and (len(args.assemblies) > 0): @@ -3524,13 +3434,13 @@ def verify_replay_common_args(): print("Don't specify `-exclude` if a collection command is given") sys.exit(1) - if ((args.pmi is True) or (args.crossgen is True) or (args.crossgen2 is True)) and (len(args.assemblies) == 0): - print("Specify `-assemblies` if `--pmi` or `--crossgen` or `--crossgen2` is given") + if ((args.pmi is True) or (args.crossgen2 is True)) and (len(args.assemblies) == 0): + print("Specify `-assemblies` if `--pmi` or `--crossgen2` is given") sys.exit(1) if args.collection_command is None and args.merge_mch_files is not True: assert args.collection_args is None - assert (args.pmi is True) or (args.crossgen is True) or (args.crossgen2 is True) + assert (args.pmi is True) or (args.crossgen2 is True) assert len(args.assemblies) > 0 if coreclr_args.merge_mch_files: