Skip to content

Commit

Permalink
Auto merge of rust-lang#129836 - jieyouxu:exp-msvc-ci-2, r=<try>
Browse files Browse the repository at this point in the history
[EXPERIMENTAL] more msvc ci debugging, do not look, cursed things inside

Messing around with msvc CI again... This time just trying to find if any processes are holding on to e.g. `miri.exe`. This is just a proof of concept, Chris probably knows how to implement this correctly lol. It uses `sysinfo` later because I got lazy and can't be bummed to also implement the find process info stuff in syscalls.

Please do not look inside as this PR contains provenance crimes and most likely UB.

This PR is not a place of honor... no highly esteemed deed is commemorated here... nothing valued is here.

What is here was dangerous and repulsive to us. This message is a warning about danger.

The danger is in a particular location... it increases towards a center... the center of danger is here... of a particular size and shape, and below us.

The danger is still present, in your time, as it was in ours.

The danger is to the body, and it can kill.

The form of the danger is an emanation of Windows API calls.

The danger is unleashed only if you substantially look at this PR. This PR is best shunned and left uninhabited.

r? ghost

try-job: x86_64-msvc-ext
  • Loading branch information
bors committed Sep 2, 2024
2 parents 9b82580 + 1152626 commit 8346ab6
Show file tree
Hide file tree
Showing 5 changed files with 234 additions and 7 deletions.
7 changes: 7 additions & 0 deletions src/bootstrap/Cargo.lock
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,12 @@ version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1"

[[package]]
name = "anyhow"
version = "1.0.86"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"

[[package]]
name = "bitflags"
version = "2.6.0"
Expand All @@ -36,6 +42,7 @@ dependencies = [
name = "bootstrap"
version = "0.0.0"
dependencies = [
"anyhow",
"build_helper",
"cc",
"clap",
Expand Down
12 changes: 11 additions & 1 deletion src/bootstrap/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ build = "build.rs"
default-run = "bootstrap"

[features]
default = ["sysinfo"]
build-metrics = ["sysinfo"]
bootstrap-self-test = [] # enabled in the bootstrap unit tests

Expand Down Expand Up @@ -41,7 +42,7 @@ cc = "=1.0.97"
cmake = "=0.1.48"

build_helper = { path = "../tools/build_helper" }
clap = { version = "4.4", default-features = false, features = ["std", "usage", "help", "derive", "error-context"] }
clap = { version = "4.4", default-features = false, features = ["derive", "error-context", "help", "std", "usage"] }
clap_complete = "4.4"
fd-lock = "4.0"
home = "0.5"
Expand All @@ -62,6 +63,9 @@ toml = "0.5"
walkdir = "2.4"
xz2 = "0.1"

# EXPERIMENTAL
anyhow = "1"

# Dependencies needed by the build-metrics feature
sysinfo = { version = "0.31.2", default-features = false, optional = true, features = ["system"] }

Expand All @@ -71,13 +75,19 @@ version = "1.0.0"
[target.'cfg(windows)'.dependencies.windows]
version = "0.52"
features = [
"Wdk_Foundation",
"Wdk_Storage_FileSystem",
"Wdk_System_SystemServices",
"Win32_Foundation",
"Win32_Security",
"Win32_Storage_FileSystem",
"Win32_System_Diagnostics_Debug",
"Win32_System_IO",
"Win32_System_JobObjects",
"Win32_System_ProcessStatus",
"Win32_System_Threading",
"Win32_System_Time",
"Win32_System_WindowsProgramming",
]

[dev-dependencies]
Expand Down
58 changes: 58 additions & 0 deletions src/bootstrap/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ use crate::utils::helpers::{

mod core;
mod utils;
#[cfg(windows)]
mod windows_hacks;

pub use core::builder::PathSet;
pub use core::config::flags::{Flags, Subcommand};
Expand Down Expand Up @@ -1663,12 +1665,68 @@ Executed at: {executed_at}"#,
if src == dst {
return;
}

if let Err(e) = fs::remove_file(dst) {
if cfg!(windows) && e.kind() != io::ErrorKind::NotFound {
// workaround for https://github.com/rust-lang/rust/issues/127126
// if removing the file fails, attempt to rename it instead.
let now = t!(SystemTime::now().duration_since(SystemTime::UNIX_EPOCH));
let _ = fs::rename(dst, format!("{}-{}", dst.display(), now.as_nanos()));

#[cfg(windows)]
{
eprintln!(
"[DEBUG]: copy_link_internal: `fs::remove_file` failed on dst=`{}`: {e}",
dst.display()
);
eprintln!("[DEBUG]: copy_link_internal: after `fs::remove_file` failed");
// HACK(jieyouxu): let's see what's holding up. Note that this is not robost to TOCTOU
// races where the process was holding on to the file when calling `remove_file` but
// released immediately after before gathering process IDs holding the file.
let mut process_ids = windows_hacks::process_ids_using_file(dst).unwrap();
process_ids.dedup();
process_ids.sort();

if !process_ids.is_empty() {
eprintln!(
"[DEBUG] copy_link_internal: pids holding dst=`{}`: {:?}",
dst.display(),
process_ids
);
}

use sysinfo::{ProcessRefreshKind, RefreshKind, System};

let sys = System::new_with_specifics(
RefreshKind::new().with_processes(ProcessRefreshKind::everything()),
);

let mut holdups = vec![];
for (pid, process) in sys.processes() {
if process_ids.contains(&(pid.as_u32() as usize)) {
holdups.push((pid.as_u32(), process.exe().unwrap_or(Path::new(""))));
}
}

if holdups.is_empty() {
eprintln!(
"[DEBUG] copy_link_internal: did not find any process holding up dst=`{}`, so how did we fail?",
dst.display(),
);
} else {
eprintln!(
"[DEBUG] copy_link_internal: printing process names (where available) holding dst=`{}`",
dst.display()
);
for (pid, process_exe) in holdups {
eprintln!(
"[DEBUG] copy_link_internal: process holding dst=`{}`: pid={pid}, process_name={:?}",
dst.display(),
process_exe
);
}
}
}
}
}
let metadata = t!(src.symlink_metadata(), format!("src = {}", src.display()));
Expand Down
152 changes: 152 additions & 0 deletions src/bootstrap/src/windows_hacks.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
//! Experimental windows hacks to try find what the hecc is holding on to the files that cannot be
//! deleted.

// Adapted from <https://stackoverflow.com/questions/67187979/how-to-call-ntopenfile> from
// Delphi for Rust :3
// Also references <https://gist.github.com/antonioCoco/9db236d6089b4b492746f7de31b21d9d>.

// SAFETY:
// YOLO.

// Windows API naming
#![allow(nonstandard_style)]
// Well because CI does deny-warnings :)
#![deny(unused_imports)]

use std::mem;
use std::os::windows::ffi::OsStrExt;
use std::path::Path;

use anyhow::Result;
use windows::core::PWSTR;
use windows::Wdk::Foundation::OBJECT_ATTRIBUTES;
use windows::Wdk::Storage::FileSystem::{
NtOpenFile, NtQueryInformationFile, FILE_OPEN_REPARSE_POINT,
};
use windows::Wdk::System::SystemServices::FILE_PROCESS_IDS_USING_FILE_INFORMATION;
use windows::Win32::Foundation::{
CloseHandle, HANDLE, STATUS_INFO_LENGTH_MISMATCH, UNICODE_STRING,
};
use windows::Win32::Storage::FileSystem::{
FILE_READ_ATTRIBUTES, FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE,
};
use windows::Win32::System::WindowsProgramming::FILE_INFORMATION_CLASS;
use windows::Win32::System::IO::IO_STATUS_BLOCK;

/// Wraps a windows API that returns [`NTSTATUS`]:
///
/// - First convert [`NTSTATUS`] to [`HRESULT`].
/// - Then convert [`HRESULT`] into a [`WinError`] with or without optional info.
macro_rules! try_syscall {
($syscall: expr) => {{
let status = $syscall;
if status.is_err() {
::anyhow::Result::Err(::windows::core::Error::from(status.to_hresult()))?;
}
}};
($syscall: expr, $additional_info: expr) => {{
let status = $syscall;
if status.is_err() {
::anyhow::Result::Err(::windows::core::Error::new(
$syscall.into(),
$additional_info.into(),
))?;
}
}};
}

pub(crate) fn process_ids_using_file(path: &Path) -> Result<Vec<usize>> {
// Gotta have it in UTF-16LE.
let mut nt_path = {
let path = std::path::absolute(path)?;
r"\??\".encode_utf16().chain(path.as_os_str().encode_wide()).collect::<Vec<u16>>()
};

let nt_path_unicode_string = UNICODE_STRING {
Length: u16::try_from(nt_path.len() * 2)?,
MaximumLength: u16::try_from(nt_path.len() * 2)?,
Buffer: PWSTR::from_raw(nt_path.as_mut_ptr()),
};

let object_attributes = OBJECT_ATTRIBUTES {
Length: mem::size_of::<OBJECT_ATTRIBUTES>() as _,
ObjectName: &nt_path_unicode_string,
..Default::default()
};

let mut io_status = IO_STATUS_BLOCK::default();
let mut handle = HANDLE::default();

// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ntifs/nf-ntifs-ntopenfile
try_syscall!(
unsafe {
NtOpenFile(
&mut handle as *mut _,
FILE_READ_ATTRIBUTES.0,
&object_attributes,
&mut io_status as *mut _,
(FILE_SHARE_READ | FILE_SHARE_DELETE | FILE_SHARE_WRITE).0,
FILE_OPEN_REPARSE_POINT.0,
)
},
"tried to open file"
);

/// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/wdm/ne-wdm-_file_information_class
// Remark: apparently windows 0.52 doesn't have this or something, it appears in at least >=
// 0.53.
const FileProcessIdsUsingFileInformation: FILE_INFORMATION_CLASS = FILE_INFORMATION_CLASS(47);

// https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/ntifs/nf-ntifs-ntqueryinformationfile
const INCREMENT: usize = 8;
let mut buf = vec![FILE_PROCESS_IDS_USING_FILE_INFORMATION::default(); INCREMENT as usize];
let mut buf_idx = 0;
let mut status = unsafe {
NtQueryInformationFile(
handle,
&mut io_status as *mut _,
buf.as_mut_ptr().cast(),
(INCREMENT * mem::size_of::<FILE_PROCESS_IDS_USING_FILE_INFORMATION>()) as u32,
FileProcessIdsUsingFileInformation,
)
};
while status == STATUS_INFO_LENGTH_MISMATCH {
buf.resize(buf.len() + INCREMENT, FILE_PROCESS_IDS_USING_FILE_INFORMATION::default());
buf_idx += INCREMENT;
status = unsafe {
NtQueryInformationFile(
handle,
&mut io_status as *mut _,
buf.as_mut_ptr()
.offset(
(buf_idx * mem::size_of::<FILE_PROCESS_IDS_USING_FILE_INFORMATION>())
as isize,
)
.cast(),
(INCREMENT * mem::size_of::<FILE_PROCESS_IDS_USING_FILE_INFORMATION>()) as u32,
FileProcessIdsUsingFileInformation,
)
};
}

let mut process_ids = vec![];

for FILE_PROCESS_IDS_USING_FILE_INFORMATION {
NumberOfProcessIdsInList,
ProcessIdList: [ptr],
} in buf
{
if NumberOfProcessIdsInList >= 1 {
// only fetch the first one
process_ids.push(unsafe {
// This is almost certaintly UB, provenance be damned
let ptr = ptr as *mut usize;
*ptr
});
}
}

try_syscall!(unsafe { CloseHandle(handle) }, "close file handle");

Ok(process_ids)
}
12 changes: 6 additions & 6 deletions src/ci/github-actions/jobs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -85,12 +85,12 @@ pr:
- image: mingw-check-tidy
continue_on_error: true
<<: *job-linux-4c
- image: x86_64-gnu-llvm-17
env:
ENABLE_GCC_CODEGEN: "1"
<<: *job-linux-16c
- image: x86_64-gnu-tools
<<: *job-linux-16c
#- image: x86_64-gnu-llvm-17
# env:
# ENABLE_GCC_CODEGEN: "1"
# <<: *job-linux-16c
#- image: x86_64-gnu-tools
# <<: *job-linux-16c

# Jobs that run when you perform a try build (@bors try)
# These jobs automatically inherit envs.try, to avoid repeating
Expand Down

0 comments on commit 8346ab6

Please sign in to comment.