Skip to content

Commit

Permalink
fix enable_ansi_support() for redirected STDOUT
Browse files Browse the repository at this point in the history
.# Discussion

Even though it is used in sample methods included in the Microsoft documentation,
the `GetStdHandle(winapi::um::winbase::STD_OUTPUT_HANDLE)` method of retrieving the
console handle is a poor choice. It will fail if STDOUT is redirected, even when a
viable console is available. This can lead to incorrectly colored STDERR when STDOUT
is redirected.

Using `CreateFile2("CONOUT$", ...)` instead succeeds even when STDOUT and/or STDERR
are redirected.

[1]: https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#samples @@ https://archive.is/PHjnN#70.5%
[2]: https://docs.microsoft.com/en-us/windows/desktop/api/fileapi/nf-fileapi-createfile2#consoles @@ https://archive.is/7BWhB#71.8%
  • Loading branch information
rivy committed May 22, 2019
1 parent af47bb5 commit 23456d5
Showing 1 changed file with 25 additions and 12 deletions.
37 changes: 25 additions & 12 deletions src/windows.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,35 +8,48 @@
/// Returns a `Result` with the Windows error code if unsuccessful.
#[cfg(windows)]
pub fn enable_ansi_support() -> Result<(), u32> {
use winapi::um::processenv::GetStdHandle;
use winapi::um::errhandlingapi::GetLastError;
// ref: https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#EXAMPLE_OF_ENABLING_VIRTUAL_TERMINAL_PROCESSING @@ https://archive.is/L7wRJ#76%

use std::ffi::OsStr;
use std::iter::once;
use std::os::windows::ffi::OsStrExt;
use std::ptr::null_mut;
use winapi::um::consoleapi::{GetConsoleMode, SetConsoleMode};
use winapi::um::errhandlingapi::GetLastError;
use winapi::um::fileapi::CreateFile2;
use winapi::um::handleapi::INVALID_HANDLE_VALUE;

const STD_OUT_HANDLE: u32 = -11i32 as u32;
const ENABLE_VIRTUAL_TERMINAL_PROCESSING: u32 = 0x0004;

unsafe {
// https://docs.microsoft.com/en-us/windows/console/getstdhandle
let std_out_handle = GetStdHandle(STD_OUT_HANDLE);
if std_out_handle == INVALID_HANDLE_VALUE
// ref: https://docs.microsoft.com/en-us/windows/desktop/api/fileapi/nf-fileapi-createfile2
// Using `CreateFile2("CONOUT$", ...)` to retrieve the console handle works correctly even if STDOUT and/or STDERR are redirected
let console_out_name: Vec<u16> = OsStr::new("CONOUT$").encode_wide().chain(once(0)).collect();
let console_handle = CreateFile2(
console_out_name.as_ptr(),
winapi::um::winnt::GENERIC_READ | winapi::um::winnt::GENERIC_WRITE,
winapi::um::winnt::FILE_SHARE_WRITE,
winapi::um::fileapi::OPEN_EXISTING,
null_mut(),
);
if console_handle == INVALID_HANDLE_VALUE
{
return Err(GetLastError());
return Err(GetLastError());
}
// https://docs.microsoft.com/en-us/windows/console/getconsolemode

// ref: https://docs.microsoft.com/en-us/windows/console/getconsolemode
let mut console_mode: u32 = 0;
if 0 == GetConsoleMode(std_out_handle, &mut console_mode)
if 0 == GetConsoleMode(console_handle, &mut console_mode)
{
return Err(GetLastError());
}

// VT processing not already enabled?
if console_mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING == 0 {
// https://docs.microsoft.com/en-us/windows/console/setconsolemode
if 0 == SetConsoleMode(std_out_handle, console_mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING)
if 0 == SetConsoleMode(console_handle, console_mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING)
{
return Err(GetLastError());
return Err(GetLastError());
}
}
}
Expand Down

0 comments on commit 23456d5

Please sign in to comment.