Auto merge of rust-lang#127760 - jieyouxu:rmake-support-reorganize, r…

…=<try>

Reorganize the `run-make-support` library

The `run_make_support` library has a kitchen sink `lib.rs` that make discovery/learning very difficult. Let's try to improve that by breaking up `lib.rs` into smaller more organized modules. This is a precursor to improving the documentation and learnability of the `run_make_support` library.

### Changes

- Breakup `lib.rs` into smaller modules according to functionality
- Rename `recursive_diff` -> `assert_recursive_eq`
- Minor doc improvements / fixes in a few places (I have a follow-up documentation PR planned)

This PR is best reviewed commit-by-commit.

r? `@Kobzol` (or Mark, or T-compiler or T-bootstrap)

try-job: x86_64-msvc
try-job: aarch64-apple
try-job: test-various
try-job: armhf-gnu
try-job: dist-x86_64-linux

src/tools/run-make-support/src/ar.rs

@@ -0,0 +1,22 @@
+use std::fs;
+use std::path::Path;
+
+/// Archive utility.
+///
+/// # Notes
+///
+/// This *currently* uses the [ar][rust-ar] crate, but this is subject to changes. We may need to
+/// use `llvm-ar`, and if that is the case, this should be moved under `external_deps`.
+///
+/// [rust-ar]: https://github.com/mdsteele/rust-ar
+#[track_caller]
+pub fn ar(inputs: &[impl AsRef<Path>], output_path: impl AsRef<Path>) {
+    let output = fs::File::create(&output_path).expect(&format!(
+        "the file in path `{}` could not be created",
+        output_path.as_ref().display()
+    ));
+    let mut builder = ar::Builder::new(output);
+    for input in inputs {
+        builder.append_path(input).unwrap();
+    }
+}

src/tools/run-make-support/src/artifact_names.rs

@@ -0,0 +1,81 @@
+//! A collection of helpers to construct artifact names, such as names of dynamic or static
+//! librarys which are target-dependent.
+
+use crate::targets::{is_darwin, is_msvc, is_windows};
+
+/// Construct the static library name based on the target.
+#[must_use]
+pub fn static_lib_name(name: &str) -> String {
+    // See tools.mk (irrelevant lines omitted):
+    //
+    // ```makefile
+    // ifeq ($(UNAME),Darwin)
+    //     STATICLIB = $(TMPDIR)/lib$(1).a
+    // else
+    //     ifdef IS_WINDOWS
+    //         ifdef IS_MSVC
+    //             STATICLIB = $(TMPDIR)/$(1).lib
+    //         else
+    //             STATICLIB = $(TMPDIR)/lib$(1).a
+    //         endif
+    //     else
+    //         STATICLIB = $(TMPDIR)/lib$(1).a
+    //     endif
+    // endif
+    // ```
+    assert!(!name.contains(char::is_whitespace), "static library name cannot contain whitespace");
+
+    if is_msvc() { format!("{name}.lib") } else { format!("lib{name}.a") }
+}
+
+/// Construct the dynamic library name based on the target.
+#[must_use]
+pub fn dynamic_lib_name(name: &str) -> String {
+    // See tools.mk (irrelevant lines omitted):
+    //
+    // ```makefile
+    // ifeq ($(UNAME),Darwin)
+    //     DYLIB = $(TMPDIR)/lib$(1).dylib
+    // else
+    //     ifdef IS_WINDOWS
+    //         DYLIB = $(TMPDIR)/$(1).dll
+    //     else
+    //         DYLIB = $(TMPDIR)/lib$(1).so
+    //     endif
+    // endif
+    // ```
+    assert!(!name.contains(char::is_whitespace), "dynamic library name cannot contain whitespace");
+
+    let extension = dynamic_lib_extension();
+    if is_darwin() {
+        format!("lib{name}.{extension}")
+    } else if is_windows() {
+        format!("{name}.{extension}")
+    } else {
+        format!("lib{name}.{extension}")
+    }
+}
+
+/// Construct the dynamic library extension based on the target.
+#[must_use]
+pub fn dynamic_lib_extension() -> &'static str {
+    if is_darwin() {
+        "dylib"
+    } else if is_windows() {
+        "dll"
+    } else {
+        "so"
+    }
+}
+
+/// Construct the name of a rust library (rlib).
+#[must_use]
+pub fn rust_lib_name(name: &str) -> String {
+    format!("lib{name}.rlib")
+}
+
+/// Construct the binary (executable) name based on the target.
+#[must_use]
+pub fn bin_name(name: &str) -> String {
+    if is_windows() { format!("{name}.exe") } else { name.to_string() }
+}

src/tools/run-make-support/src/assertion_helpers.rs

@@ -0,0 +1,168 @@
+//! Collection of assertions and assertion-related helpers.
+
+use std::panic;
+use std::path::{Path, PathBuf};
+
+use crate::fs_helpers;
+use crate::fs_wrapper;
+use crate::path_helpers::cwd;
+
+/// Browse the directory `path` non-recursively and return all files which respect the parameters
+/// outlined by `closure`.
+#[track_caller]
+pub fn shallow_find_files<P: AsRef<Path>, F: Fn(&PathBuf) -> bool>(
+    path: P,
+    filter: F,
+) -> Vec<PathBuf> {
+    let mut matching_files = Vec::new();
+    for entry in fs_wrapper::read_dir(path) {
+        let entry = entry.expect("failed to read directory entry.");
+        let path = entry.path();
+
+        if path.is_file() && filter(&path) {
+            matching_files.push(path);
+        }
+    }
+    matching_files
+}
+
+/// Returns true if the filename at `path` starts with `prefix`.
+pub fn has_prefix<P: AsRef<Path>>(path: P, prefix: &str) -> bool {
+    path.as_ref().file_name().is_some_and(|name| name.to_str().unwrap().starts_with(prefix))
+}
+
+/// Returns true if the filename at `path` has the extension `extension`.
+pub fn has_extension<P: AsRef<Path>>(path: P, extension: &str) -> bool {
+    path.as_ref().extension().is_some_and(|ext| ext == extension)
+}
+
+/// Returns true if the filename at `path` does not contain `expected`.
+pub fn not_contains<P: AsRef<Path>>(path: P, expected: &str) -> bool {
+    !path.as_ref().file_name().is_some_and(|name| name.to_str().unwrap().contains(expected))
+}
+
+/// Returns true if the filename at `path` is not in `expected`.
+pub fn filename_not_in_denylist<P: AsRef<Path>, V: AsRef<[String]>>(path: P, expected: V) -> bool {
+    let expected = expected.as_ref();
+    path.as_ref()
+        .file_name()
+        .is_some_and(|name| !expected.contains(&name.to_str().unwrap().to_owned()))
+}
+
+/// Returns true if the filename at `path` ends with `suffix`.
+pub fn has_suffix<P: AsRef<Path>>(path: P, suffix: &str) -> bool {
+    path.as_ref().file_name().is_some_and(|name| name.to_str().unwrap().ends_with(suffix))
+}
+
+/// Gathers all files in the current working directory that have the extension `ext`, and counts
+/// the number of lines within that contain a match with the regex pattern `re`.
+pub fn count_regex_matches_in_files_with_extension(re: &regex::Regex, ext: &str) -> usize {
+    let fetched_files = shallow_find_files(cwd(), |path| has_extension(path, ext));
+
+    let mut count = 0;
+    for file in fetched_files {
+        let content = fs_wrapper::read_to_string(file);
+        count += content.lines().filter(|line| re.is_match(&line)).count();
+    }
+
+    count
+}
+
+/// Read the contents of a file that cannot simply be read by
+/// [`read_to_string`][crate::fs_wrapper::read_to_string], due to invalid UTF-8 data, then assert
+/// that it contains `expected`.
+#[track_caller]
+pub fn invalid_utf8_contains<P: AsRef<Path>, S: AsRef<str>>(path: P, expected: S) {
+    let buffer = fs_wrapper::read(path.as_ref());
+    let expected = expected.as_ref();
+    if !String::from_utf8_lossy(&buffer).contains(expected) {
+        eprintln!("=== FILE CONTENTS (LOSSY) ===");
+        eprintln!("{}", String::from_utf8_lossy(&buffer));
+        eprintln!("=== SPECIFIED TEXT ===");
+        eprintln!("{}", expected);
+        panic!("specified text was not found in file");
+    }
+}
+
+/// Read the contents of a file that cannot simply be read by
+/// [`read_to_string`][crate::fs_wrapper::read_to_string], due to invalid UTF-8 data, then assert
+/// that it does not contain `expected`.
+#[track_caller]
+pub fn invalid_utf8_not_contains<P: AsRef<Path>, S: AsRef<str>>(path: P, expected: S) {
+    let buffer = fs_wrapper::read(path.as_ref());
+    let expected = expected.as_ref();
+    if String::from_utf8_lossy(&buffer).contains(expected) {
+        eprintln!("=== FILE CONTENTS (LOSSY) ===");
+        eprintln!("{}", String::from_utf8_lossy(&buffer));
+        eprintln!("=== SPECIFIED TEXT ===");
+        eprintln!("{}", expected);
+        panic!("specified text was unexpectedly found in file");
+    }
+}
+
+/// Assert that `actual` is equal to `expected`.
+#[track_caller]
+pub fn assert_equals<A: AsRef<str>, E: AsRef<str>>(actual: A, expected: E) {
+    let actual = actual.as_ref();
+    let expected = expected.as_ref();
+    if actual != expected {
+        eprintln!("=== ACTUAL TEXT ===");
+        eprintln!("{}", actual);
+        eprintln!("=== EXPECTED ===");
+        eprintln!("{}", expected);
+        panic!("expected text was not found in actual text");
+    }
+}
+
+/// Assert that `haystack` contains `needle`.
+#[track_caller]
+pub fn assert_contains<H: AsRef<str>, N: AsRef<str>>(haystack: H, needle: N) {
+    let haystack = haystack.as_ref();
+    let needle = needle.as_ref();
+    if !haystack.contains(needle) {
+        eprintln!("=== HAYSTACK ===");
+        eprintln!("{}", haystack);
+        eprintln!("=== NEEDLE ===");
+        eprintln!("{}", needle);
+        panic!("needle was not found in haystack");
+    }
+}
+
+/// Assert that `haystack` does not contain `needle`.
+#[track_caller]
+pub fn assert_not_contains<H: AsRef<str>, N: AsRef<str>>(haystack: H, needle: N) {
+    let haystack = haystack.as_ref();
+    let needle = needle.as_ref();
+    if haystack.contains(needle) {
+        eprintln!("=== HAYSTACK ===");
+        eprintln!("{}", haystack);
+        eprintln!("=== NEEDLE ===");
+        eprintln!("{}", needle);
+        panic!("needle was unexpectedly found in haystack");
+    }
+}
+
+/// Assert that all files in `dir1` exist and have the same content in `dir2`
+pub fn assert_recursive_eq(dir1: impl AsRef<Path>, dir2: impl AsRef<Path>) {
+    let dir2 = dir2.as_ref();
+    fs_helpers::read_dir(dir1, |entry_path| {
+        let entry_name = entry_path.file_name().unwrap();
+        if entry_path.is_dir() {
+            assert_recursive_eq(&entry_path, &dir2.join(entry_name));
+        } else {
+            let path2 = dir2.join(entry_name);
+            let file1 = fs_wrapper::read(&entry_path);
+            let file2 = fs_wrapper::read(&path2);
+
+            // We don't use `assert_eq!` because they are `Vec<u8>`, so not great for display.
+            // Why not using String? Because there might be minified files or even potentially
+            // binary ones, so that would display useless output.
+            assert!(
+                file1 == file2,
+                "`{}` and `{}` have different content",
+                entry_path.display(),
+                path2.display(),
+            );
+        }
+    });
+}

src/tools/run-make-support/src/command.rs

@@ -5,7 +5,9 @@ use std::panic;
 use std::path::Path;
 use std::process::{Command as StdCommand, ExitStatus, Output, Stdio};
 
-use crate::{assert_contains, assert_equals, assert_not_contains, handle_failed_output};
+use crate::util::handle_failed_output;
+use crate::{assert_contains, assert_equals, assert_not_contains};
+
 use build_helper::drop_bomb::DropBomb;
 
 /// This is a custom command wrapper that simplifies working with commands and makes it easier to

src/tools/run-make-support/src/env_checked.rs

@@ -0,0 +1,20 @@
+use std::env;
+use std::ffi::OsString;
+
+#[track_caller]
+#[must_use]
+pub fn env_var(name: &str) -> String {
+    match env::var(name) {
+        Ok(v) => v,
+        Err(err) => panic!("failed to retrieve environment variable {name:?}: {err:?}"),
+    }
+}
+
+#[track_caller]
+#[must_use]
+pub fn env_var_os(name: &str) -> OsString {
+    match env::var_os(name) {
+        Some(v) => v,
+        None => panic!("failed to retrieve environment variable {name:?}"),
+    }
+}

src/tools/run-make-support/src/cc.rs → src/tools/run-make-support/src/external_deps/cc.rs

@@ -20,7 +20,7 @@ pub struct Cc {
     cmd: Command,
 }
 
-crate::impl_common_helpers!(Cc);
+crate::macros::impl_common_helpers!(Cc);
 
 impl Cc {
     /// Construct a new platform-specific C compiler invocation.

src/tools/run-make-support/src/clang.rs → src/tools/run-make-support/src/external_deps/clang.rs

@@ -16,7 +16,7 @@ pub struct Clang {
     cmd: Command,
 }
 
-crate::impl_common_helpers!(Clang);
+crate::macros::impl_common_helpers!(Clang);
 
 impl Clang {
     /// Construct a new `clang` invocation. `clang` is not always available for all targets.

src/tools/run-make-support/src/external_deps/htmldocck.rs

@@ -0,0 +1,14 @@
+use crate::command::Command;
+use crate::source_root;
+
+use super::python::python_command;
+
+/// `htmldocck` is a python script which is used for rustdoc test suites, it is assumed to be
+/// available at `$SOURCE_ROOT/src/etc/htmldocck.py`.
+#[track_caller]
+#[must_use]
+pub fn htmldocck() -> Command {
+    let mut python = python_command();
+    python.arg(source_root().join("src/etc/htmldocck.py"));
+    python
+}

src/tools/run-make-support/src/llvm.rs → src/tools/run-make-support/src/external_deps/llvm.rs

@@ -1,6 +1,7 @@
 use std::path::{Path, PathBuf};
 
-use crate::{env_var, Command};
+use crate::command::Command;
+use crate::env_checked::env_var;
 
 /// Construct a new `llvm-readobj` invocation with the `GNU` output style.
 /// This assumes that `llvm-readobj` is available at `$LLVM_BIN_DIR/llvm-readobj`.
@@ -57,10 +58,10 @@ pub struct LlvmObjdump {
     cmd: Command,
 }
 
-crate::impl_common_helpers!(LlvmReadobj);
-crate::impl_common_helpers!(LlvmProfdata);
-crate::impl_common_helpers!(LlvmFilecheck);
-crate::impl_common_helpers!(LlvmObjdump);
+crate::macros::impl_common_helpers!(LlvmReadobj);
+crate::macros::impl_common_helpers!(LlvmProfdata);
+crate::macros::impl_common_helpers!(LlvmFilecheck);
+crate::macros::impl_common_helpers!(LlvmObjdump);
 
 /// Generate the path to the bin directory of LLVM.
 #[must_use]

src/tools/run-make-support/src/external_deps/mod.rs

@@ -0,0 +1,15 @@
+//! This module contains external tool dependencies that we assume are available in the environment,
+//! such as `cc` or `python`.
+//!
+//! # Notes
+//!
+//! - This is not the *only* place where external dependencies are assumed or referenced. For
+//!   example, see [`cygpath_windows`][crate::path_helpers::cygpath_windows].
+
+pub mod cc;
+pub mod clang;
+pub mod htmldocck;
+pub mod llvm;
+pub mod python;
+pub mod rustc;
+pub mod rustdoc;

src/tools/run-make-support/src/external_deps/python.rs

@@ -0,0 +1,11 @@
+use crate::command::Command;
+use crate::env_checked::env_var;
+
+/// Obtain path of python as provided by the `PYTHON` environment variable. It is up to the caller
+/// to document and check if the python version is compatible with its intended usage.
+#[track_caller]
+#[must_use]
+pub fn python_command() -> Command {
+    let python_path = env_var("PYTHON");
+    Command::new(python_path)
+}