Make x.py clippy download and use beta clippy

Cargo.lock

@@ -359,7 +359,6 @@ dependencies = [
  "libgit2-sys",
  "log",
  "memchr",
- "num_cpus",
  "opener",
  "openssl",
  "os_info",
@@ -471,7 +470,7 @@ dependencies = [
 
 [[package]]
 name = "cargo-util"
-version = "0.1.4"
+version = "0.1.3"
 dependencies = [
  "anyhow",
  "core-foundation",
@@ -5556,8 +5555,6 @@ dependencies = [
  "pretty_assertions 1.2.1",
  "regex",
  "rustc_version",
- "serde",
- "serde_json",
 ]
 
 [[package]]

src/bootstrap/bin/rustc.rs

@@ -36,14 +36,25 @@ fn main() {
         Err(_) => 0,
     };
 
+    if verbose > 1 {
+        eprintln!("target: {target:?}");
+        eprintln!("version: {version:?}");
+    }
+
     // Use a different compiler for build scripts, since there may not yet be a
     // libstd for the real compiler to use. However, if Cargo is attempting to
     // determine the version of the compiler, the real compiler needs to be
     // used. Currently, these two states are differentiated based on whether
     // --target and -vV is/isn't passed.
     let (rustc, libdir) = if target.is_none() && version.is_none() {
+        if verbose > 1 {
+            eprintln!("Using snapshot complier");
+        }
         ("RUSTC_SNAPSHOT", "RUSTC_SNAPSHOT_LIBDIR")
     } else {
+        if verbose > 1 {
+            eprintln!("Using real complier");
+        }
         ("RUSTC_REAL", "RUSTC_LIBDIR")
     };
     let stage = env::var("RUSTC_STAGE").expect("RUSTC_STAGE was not set");
@@ -70,6 +81,10 @@ fn main() {
                 cmd.arg("-Ztime");
             }
         }
+
+        if crate_name == "build_script_build" {
+            eprintln!("building build scripts using sysroot {:?}", sysroot);
+        }
     }
 
     // Print backtrace in case of ICE
@@ -141,6 +156,15 @@ fn main() {
         cmd.arg("--check-cfg=values(bootstrap)");
     }
 
+    if let Ok(command) = env::var("RUSTC_COMMAND") {
+        if command == "clippy" && target.is_none() {
+            let libdir_string = libdir.to_string_lossy();
+            let (sysroot, _) = libdir_string.rsplit_once('/').unwrap();
+            eprintln!("passing clippy --sysroot {}", sysroot);
+            cmd.arg("--sysroot").arg(&sysroot);
+        }
+    }
+
     if let Ok(map) = env::var("RUSTC_DEBUGINFO_MAP") {
         cmd.arg("--remap-path-prefix").arg(&map);
     }
@@ -179,6 +203,7 @@ fn main() {
             env::join_paths(&dylib_path).unwrap(),
             cmd,
         );
+        eprintln!("{} SYSROOT: {:?}", prefix, env::var("SYSROOT"));
         eprintln!("{} sysroot: {:?}", prefix, sysroot);
         eprintln!("{} libdir: {:?}", prefix, libdir);
     }

src/bootstrap/bootstrap.py

@@ -454,13 +454,14 @@ def download_toolchain(self):
         rustc_channel = self.stage0_compiler.version
         bin_root = self.bin_root()
 
+        tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
+
         key = self.stage0_compiler.date
         if self.rustc().startswith(bin_root) and \
                 (not os.path.exists(self.rustc()) or
                  self.program_out_of_date(self.rustc_stamp(), key)):
             if os.path.exists(bin_root):
                 shutil.rmtree(bin_root)
-            tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
             filename = "rust-std-{}-{}{}".format(
                 rustc_channel, self.build, tarball_suffix)
             pattern = "rust-std-{}".format(self.build)
@@ -482,6 +483,28 @@ def download_toolchain(self):
             with output(self.rustc_stamp()) as rust_stamp:
                 rust_stamp.write(key)
 
+        clippy_path = self.clippy()
+        clippy_needs_download = not os.path.exists(clippy_path) \
+            or self.program_out_of_date(self.clippy_stamp(), key)
+        if clippy_path.startswith(bin_root) and clippy_needs_download:
+            # download Clippy
+            # the component name is clippy, but the bin containing folder name is clippy-preview
+            filename = self._format_component_filename(
+                "clippy",
+                rustc_channel,
+                self.build,
+                tarball_suffix
+            )
+            self._download_component_helper(filename, "clippy-preview", tarball_suffix)
+            self.fix_bin_or_dylib("{}/bin/clippy-driver".format(bin_root))
+            self.fix_bin_or_dylib("{}/bin/cargo-clippy".format(bin_root))
+
+            with output(self.clippy_stamp()) as clippy_stamp:
+                clippy_stamp.write(key)
+
+    def _format_component_filename(self, component_name, version, build, tarball_suffix):
+        return "{}-{}-{}{}".format(component_name, version, build, tarball_suffix)
+
     def _download_component_helper(
         self, filename, pattern, tarball_suffix,
     ):
@@ -610,6 +633,27 @@ def rustc_stamp(self):
         """
         return os.path.join(self.bin_root(), '.rustc-stamp')
 
+    def rustfmt_stamp(self):
+        """Return the path for .rustfmt-stamp
+
+        >>> rb = RustBuild()
+        >>> rb.build_dir = "build"
+        >>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
+        True
+        """
+        return os.path.join(self.bin_root(), '.rustfmt-stamp')
+
+    def clippy_stamp(self):
+        """Return the path for .clippy-stamp
+
+        >>> rb = RustBuild()
+        >>> rb.build_dir = "build"
+        >>> rb.clippy_stamp() == os.path.join("build", "stage0", ".clippy-stamp")
+        True
+        """
+        return os.path.join(self.bin_root(), '.clippy-stamp')
+
+
     def program_out_of_date(self, stamp_path, key):
         """Check if the given program stamp is out of date"""
         if not os.path.exists(stamp_path) or self.clean:
@@ -683,6 +727,10 @@ def rustc(self):
         """Return config path for rustc"""
         return self.program_config('rustc')
 
+    def clippy(self):
+        """Return config path for clippy"""
+        return self.program_config('cargo-clippy')
+
     def program_config(self, program):
         """Return config path for the given program at the given stage
 

src/bootstrap/builder.rs

@@ -1310,7 +1310,12 @@ impl<'a> Builder<'a> {
         target: TargetSelection,
         cmd: &str,
     ) -> Cargo {
-        let mut cargo = Command::new(&self.initial_cargo);
+        let mut cargo = if cmd == "clippy" {
+            Command::new(self.initial_rustc.parent().unwrap().join("cargo-clippy"))
+        } else {
+            Command::new(&self.initial_cargo)
+        };
+
         let out_dir = self.stage_out(compiler, mode);
 
         // Codegen backends are not yet tracked by -Zbinary-dep-depinfo,
@@ -1391,6 +1396,22 @@ impl<'a> Builder<'a> {
             compiler.stage
         };
 
+        // We synthetically interpret a stage0 compiler used to build tools as a
+        // "raw" compiler in that it's the exact snapshot we download. Normally
+        // the stage0 build means it uses libraries build by the stage0
+        // compiler, but for tools we just use the precompiled libraries that
+        // we've downloaded
+        let use_snapshot = mode == Mode::ToolBootstrap;
+        assert!(!use_snapshot || stage == 0 || self.local_rebuild);
+
+        let maybe_sysroot = self.sysroot(compiler);
+        let sysroot = if use_snapshot { self.rustc_snapshot_sysroot() } else { &maybe_sysroot };
+        let libdir = self.rustc_libdir(compiler);
+
+        let sysroot_str = sysroot.as_os_str().to_str().expect("sysroot should be UTF-8");
+        self.verbose_than(0, &format!("using sysroot {sysroot_str}"));
+        self.verbose_than(1, &format!("running cargo with mode {mode:?}"));
+
         let mut rustflags = Rustflags::new(target);
         if stage != 0 {
             if let Ok(s) = env::var("CARGOFLAGS_NOT_BOOTSTRAP") {
@@ -1408,35 +1429,12 @@ impl<'a> Builder<'a> {
                 // NOTE: this can't be fixed in clippy because we explicitly don't set `RUSTC`,
                 // so it has no way of knowing the sysroot.
                 rustflags.arg("--sysroot");
-                rustflags.arg(
-                    self.sysroot(compiler)
-                        .as_os_str()
-                        .to_str()
-                        .expect("sysroot must be valid UTF-8"),
-                );
+                rustflags.arg(sysroot_str);
                 // Only run clippy on a very limited subset of crates (in particular, not build scripts).
                 cargo.arg("-Zunstable-options");
-                // Explicitly does *not* set `--cfg=bootstrap`, since we're using a nightly clippy.
-                let host_version = Command::new("rustc").arg("--version").output().map_err(|_| ());
-                let output = host_version.and_then(|output| {
-                    if output.status.success() {
-                        Ok(output)
-                    } else {
-                        Err(())
-                    }
-                }).unwrap_or_else(|_| {
-                    eprintln!(
-                        "error: `x.py clippy` requires a host `rustc` toolchain with the `clippy` component"
-                    );
-                    eprintln!("help: try `rustup component add clippy`");
-                    std::process::exit(1);
-                });
-                if !t!(std::str::from_utf8(&output.stdout)).contains("nightly") {
-                    rustflags.arg("--cfg=bootstrap");
-                }
-            } else {
-                rustflags.arg("--cfg=bootstrap");
             }
+
+            rustflags.arg("--cfg=bootstrap");
         }
 
         let use_new_symbol_mangling = match self.config.rust_new_symbol_mangling {
@@ -1556,6 +1554,10 @@ impl<'a> Builder<'a> {
             Mode::Std | Mode::Rustc | Mode::Codegen | Mode::ToolRustc => {}
         }
 
+        if self.jobs() > 1 {
+            //panic!("TESTING: Run with one job only!");
+        }
+
         cargo.arg("-j").arg(self.jobs().to_string());
         // Remove make-related flags to ensure Cargo can correctly set things up
         cargo.env_remove("MAKEFLAGS");
@@ -1609,18 +1611,6 @@ impl<'a> Builder<'a> {
 
         let want_rustdoc = self.doc_tests != DocTests::No;
 
-        // We synthetically interpret a stage0 compiler used to build tools as a
-        // "raw" compiler in that it's the exact snapshot we download. Normally
-        // the stage0 build means it uses libraries build by the stage0
-        // compiler, but for tools we just use the precompiled libraries that
-        // we've downloaded
-        let use_snapshot = mode == Mode::ToolBootstrap;
-        assert!(!use_snapshot || stage == 0 || self.local_rebuild);
-
-        let maybe_sysroot = self.sysroot(compiler);
-        let sysroot = if use_snapshot { self.rustc_snapshot_sysroot() } else { &maybe_sysroot };
-        let libdir = self.rustc_libdir(compiler);
-
         // Clear the output directory if the real rustc we're using has changed;
         // Cargo cannot detect this as it thinks rustc is bootstrap/debug/rustc.
         //
@@ -1643,6 +1633,11 @@ impl<'a> Builder<'a> {
             .env("RUSTBUILD_NATIVE_DIR", self.native_dir(target))
             .env("RUSTC_REAL", self.rustc(compiler))
             .env("RUSTC_STAGE", stage.to_string())
+
+            // set for clippy to know the sysroot
+            .env("SYSROOT", &sysroot)
+            .env("RUSTC_COMMAND", cmd)
+
             .env("RUSTC_SYSROOT", &sysroot)
             .env("RUSTC_LIBDIR", &libdir)
             .env("RUSTDOC", self.bootstrap_out.join("rustdoc"))
@@ -1656,11 +1651,8 @@ impl<'a> Builder<'a> {
             )
             .env("RUSTC_ERROR_METADATA_DST", self.extended_error_dir())
             .env("RUSTC_BREAK_ON_ICE", "1");
-        // Clippy support is a hack and uses the default `cargo-clippy` in path.
-        // Don't override RUSTC so that the `cargo-clippy` in path will be run.
-        if cmd != "clippy" {
-            cargo.env("RUSTC", self.bootstrap_out.join("rustc"));
-        }
+
+        cargo.env("RUSTC", self.bootstrap_out.join("rustc"));
 
         // Dealing with rpath here is a little special, so let's go into some
         // detail. First off, `-rpath` is a linker option on Unix platforms

src/bootstrap/flags.rs

@@ -81,6 +81,7 @@ pub struct Flags {
 }
 
 #[cfg_attr(test, derive(Clone))]
+#[derive(Debug)]
 pub enum Subcommand {
     Build {
         paths: Vec<PathBuf>,

src/bootstrap/setup.rs

@@ -11,7 +11,7 @@ use std::{
     io::{self, Write},
 };
 
-#[derive(Clone, Copy, Eq, PartialEq)]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
 pub enum Profile {
     Compiler,
     Codegen,

src/tools/bump-stage0/src/main.rs

@@ -5,7 +5,7 @@ use std::collections::HashMap;
 use std::convert::TryInto;
 
 const PATH: &str = "src/stage0.json";
-const COMPILER_COMPONENTS: &[&str] = &["rustc", "rust-std", "cargo"];
+const COMPILER_COMPONENTS: &[&str] = &["rustc", "rust-std", "cargo", "clippy-preview"];
 const RUSTFMT_COMPONENTS: &[&str] = &["rustfmt-preview"];
 
 struct Tool {