From 93053d801990df3bdd6377afe2c137af54b68244 Mon Sep 17 00:00:00 2001 From: csmoe Date: Fri, 21 Apr 2023 18:21:20 +0800 Subject: [PATCH] enable PGO on x86_64-apple-darwin --- .github/workflows/ci.yml | 2 +- src/bootstrap/compile.rs | 1 + src/bootstrap/llvm.rs | 8 +++--- src/bootstrap/util.rs | 28 +++++++-------------- src/ci/github-actions/ci.yml | 2 +- src/ci/scripts/install-clang.sh | 13 +++++++--- src/ci/stage-build.py | 43 +++++++++++++++++++++++++++++++++ 7 files changed, 69 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dacf929278ab9..13a2be1ff0fd9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -317,7 +317,7 @@ jobs: os: ubuntu-20.04-8core-32gb - name: dist-x86_64-apple env: - SCRIPT: "./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin" + SCRIPT: PGO_HOST=x86_64-apple-darwin python3 src/ci/stage-build.py python3 ./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin RUST_CONFIGURE_ARGS: "--enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false --set rust.lto=thin" RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 2a7dd4d1849ae..4dd74f35edc2f 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -950,6 +950,7 @@ fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelect } else { panic!("clang has no clang_rt.profile library for {target}"); }; + let clang = builder.cc(target); let clang_rt_dir = get_clang_rt_dir(clang, false); let clang_rt_profile_lib = format!("libclang_rt.profile_{clang_rt_profile_lib_suffix}"); diff --git a/src/bootstrap/llvm.rs b/src/bootstrap/llvm.rs index a6de5ddcaac9e..3aac613267e57 100644 --- a/src/bootstrap/llvm.rs +++ b/src/bootstrap/llvm.rs @@ -331,7 +331,10 @@ impl Step for Llvm { // This flag makes sure `FileCheck` is copied in the final binaries directory. cfg.define("LLVM_INSTALL_UTILS", "ON"); + let mut enabled_llvm_projects = Vec::new(); + if builder.config.llvm_profile_generate { + enabled_llvm_projects.push("compiler-rt"); cfg.define("LLVM_BUILD_INSTRUMENTED", "IR"); if let Ok(llvm_profile_dir) = std::env::var("LLVM_PROFILE_DIR") { cfg.define("LLVM_PROFILE_DATA_DIR", llvm_profile_dir); @@ -344,6 +347,7 @@ impl Step for Llvm { if builder.config.llvm_bolt_profile_generate || builder.config.llvm_bolt_profile_use.is_some() { + enabled_llvm_projects.push("bolt"); // Relocations are required for BOLT to work. ldflags.push_all("-Wl,-q"); } @@ -403,8 +407,6 @@ impl Step for Llvm { cfg.define("LLVM_BUILD_32_BITS", "ON"); } - let mut enabled_llvm_projects = Vec::new(); - if util::forcing_clang_based_tests() { enabled_llvm_projects.push("clang"); enabled_llvm_projects.push("compiler-rt"); @@ -828,7 +830,7 @@ impl Step for Lld { if let Some(clang_cl_path) = builder.config.llvm_clang_cl.as_ref() { // Find clang's runtime library directory and push that as a search path to the // cmake linker flags. - let clang_rt_dir = get_clang_rt_dir(clang_cl_path); + let clang_rt_dir = get_clang_rt_dir(clang_cl_path, true); ldflags.push_all(&format!("/libpath:{}", clang_rt_dir.display())); } } diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs index 7633f8fdb54b2..de1e2cd1fc749 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/util.rs @@ -488,32 +488,22 @@ fn absolute_windows(path: &std::path::Path) -> std::io::Result, is_msvc: bool) -> PathBuf { - // Similar to how LLVM does it, to find clang's library runtime directory: - // - we ask `clang-cl` to locate the `clang_rt.builtins` lib. - let mut builtins_locator = Command::new(clang.as_ref()); + let mut cmd = Command::new(clang.as_ref()); if is_msvc { - builtins_locator.args(&["/clang:-print-libgcc-file-name", "/clang:--rtlib=compiler-rt"]); + cmd.args(&["/clang:-print-runtime-dir"]); } else { - builtins_locator.args(&["-print-libgcc-file-name", "-rtlib=compiler-rt"]); + cmd.args(&["-print-runtime-dir"]); }; - let clang_rt_builtins = output(&mut builtins_locator); - let clang_rt_builtins = Path::new(clang_rt_builtins.trim()); + let dir = output(&mut cmd); + let dir = PathBuf::from(dir.trim()); assert!( - clang_rt_builtins.exists(), - "`clang-cl` must correctly locate the library runtime directory" + dir.exists(), + "`{}` must correctly locate the library runtime directory", + clang.as_ref().display() ); - - // - the profiler runtime will be located in the same directory as the builtins lib, like - // `$LLVM_DISTRO_ROOT/lib/clang/$LLVM_VERSION/lib/windows`. - let clang_rt_dir = clang_rt_builtins.parent().expect("The clang lib folder should exist"); - clang_rt_dir.to_path_buf() + dir } pub fn lld_flag_no_threads(is_windows: bool) -> &'static str { diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 8c7798aad8bdf..db898bccafebb 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -491,7 +491,7 @@ jobs: - name: dist-x86_64-apple env: - SCRIPT: ./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin + SCRIPT: PGO_HOST=x86_64-apple-darwin python3 src/ci/stage-build.py python3 ./x.py dist bootstrap --include-default-paths --host=x86_64-apple-darwin --target=x86_64-apple-darwin RUST_CONFIGURE_ARGS: --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false --set rust.lto=thin RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 diff --git a/src/ci/scripts/install-clang.sh b/src/ci/scripts/install-clang.sh index 02b72625d6eb5..45183391e16f6 100755 --- a/src/ci/scripts/install-clang.sh +++ b/src/ci/scripts/install-clang.sh @@ -18,10 +18,15 @@ if isMacOS; then if [[ ${USE_XCODE_CLANG-0} -eq 1 ]]; then bindir="$(xcode-select --print-path)/Toolchains/XcodeDefault.xctoolchain/usr/bin" else - file="${MIRRORS_BASE}/clang%2Bllvm-${LLVM_VERSION}-x86_64-apple-darwin.tar.xz" - retry curl -f "${file}" -o "clang+llvm-${LLVM_VERSION}-x86_64-apple-darwin.tar.xz" - tar xJf "clang+llvm-${LLVM_VERSION}-x86_64-apple-darwin.tar.xz" - bindir="$(pwd)/clang+llvm-${LLVM_VERSION}-x86_64-apple-darwin/bin" + OSX_ARCH=$(uname -m) + file="${MIRRORS_BASE}/clang%2Bllvm-${LLVM_VERSION}-${OSX_ARCH}-apple-darwin.tar.xz" + retry curl -f "${file}" -o "clang+llvm-${LLVM_VERSION}-${OSX_ARCH}-apple-darwin.tar.xz" + + mkdir -p citools && cd citools + + tar xJf "clang+llvm-${LLVM_VERSION}-${OSX_ARCH}-apple-darwin.tar.xz" + mv "clang+llvm-${LLVM_VERSION}-${OSX_ARCH}-apple-darwin" clang-rust + bindir="$(pwd)/clang-rust/bin" fi ciCommandSetEnv CC "${bindir}/clang" diff --git a/src/ci/stage-build.py b/src/ci/stage-build.py index 7cd5e88f6a285..ed54676b0852f 100644 --- a/src/ci/stage-build.py +++ b/src/ci/stage-build.py @@ -207,6 +207,47 @@ def rustc_profile_template_path(self) -> Path: def supports_bolt(self) -> bool: return False +class DarwinPipeline(Pipeline): + def __init__(self): + self.checkout_dir = Path(os.getcwd()) + + def checkout_path(self) -> Path: + return self.checkout_dir + + def downloaded_llvm_dir(self) -> Path: + return self.checkout_path() / "citools" / "clang-rust" + + def build_root(self) -> Path: + return self.checkout_path() + + def opt_artifacts(self) -> Path: + return Path("/tmp/tmp-multistage/opt-artifacts") + + def build_rustc_perf(self): + # rustc-perf version from 2022-07-22 + perf_commit = "9dfaa35193154b690922347ee1141a06ec87a199" + rustc_perf_zip_path = self.opt_artifacts() / "perf.zip" + + def download_rustc_perf(): + download_file( + f"https://github.com/rust-lang/rustc-perf/archive/{perf_commit}.zip", + rustc_perf_zip_path + ) + with change_cwd(self.opt_artifacts()): + unpack_archive(rustc_perf_zip_path) + move_path(Path(f"rustc-perf-{perf_commit}"), self.rustc_perf_dir()) + delete_file(rustc_perf_zip_path) + + retry_action(download_rustc_perf, "Download rustc-perf") + + with change_cwd(self.rustc_perf_dir()): + cmd([self.cargo_stage_0(), "build", "-p", "collector"], env=dict( + RUSTC=str(self.rustc_stage_0()), + RUSTC_BOOTSTRAP="1" + )) + + def supports_bolt(self) -> bool: + return False def get_timestamp() -> float: return time.time() @@ -576,6 +617,8 @@ def create_pipeline() -> Pipeline: return LinuxPipeline() elif sys.platform in ("cygwin", "win32"): return WindowsPipeline() + elif sys.platform == "darwin": + return DarwinPipeline() else: raise Exception(f"Optimized build is not supported for platform {sys.platform}")