windows: stub out a winpty wrapper

conpty, while it may be the best native API available for consoles
built in to Windows 10, currently swallows mouse escape sequences
(microsoft/terminal#376) so we need an
alternative for a better windows experience.

winpty appears to be a reasonably mature pty implementation, so
let's try that!

I tried to use winpty-sys from crates.io but it requires the
installation of an external clang compiler so I used bindgen
on my linux box to convert the small header file to rust and
then tweaked it a bit.

This commit includes that basic wrapper plus a type safe layer
on top.

This will require distributing the dll and an agent helper along
with the wezterm.exe.

This commit doesn't implement the wezterm side of the pty interface;
that will be in a follow up commit.

src/pty/conpty.rs

@@ -397,7 +397,7 @@ impl PsuedoCon {
 }
 
 #[derive(Debug)]
-struct OwnedHandle {
+pub struct OwnedHandle {
     handle: HANDLE,
 }
 unsafe impl Send for OwnedHandle {}
@@ -410,7 +410,10 @@ impl Drop for OwnedHandle {
 }
 
 impl OwnedHandle {
-    fn try_clone(&self) -> Result<Self, IoError> {
+    pub fn new(handle: HANDLE) -> Self {
+        Self { handle }
+    }
+    pub fn try_clone(&self) -> Result<Self, IoError> {
         if self.handle == INVALID_HANDLE_VALUE || self.handle.is_null() {
             return Ok(OwnedHandle {
                 handle: self.handle,

src/pty/mod.rs

@@ -2,6 +2,8 @@
 pub mod conpty;
 #[cfg(unix)]
 pub mod unix;
+#[cfg(windows)]
+pub mod winpty;
 
 #[cfg(windows)]
 pub use self::conpty::{openpty, Child, Command, ExitStatus, MasterPty, SlavePty};

src/pty/winpty/mod.rs

@@ -0,0 +1,2 @@
+mod safe;
+mod sys;

src/pty/winpty/safe.rs

@@ -0,0 +1,263 @@
+//! A type-safe wrapper around the sys module, which in turn exposes
+//! the API exported by winpty.dll.
+//! https://github.com/rprichard/winpty/blob/master/src/include/winpty.h
+use super::sys::*;
+use crate::pty::conpty::OwnedHandle;
+use bitflags::bitflags;
+use failure::{format_err, Error};
+use std::ffi::{OsStr, OsString};
+use std::os::windows::ffi::{OsStrExt, OsStringExt};
+use std::ptr;
+use winapi::shared::minwindef::DWORD;
+use winapi::shared::ntdef::LPCWSTR;
+use winapi::um::fileapi::{CreateFileW, OPEN_EXISTING};
+use winapi::um::handleapi::INVALID_HANDLE_VALUE;
+use winapi::um::winbase::INFINITE;
+use winapi::um::winnt::HANDLE;
+use winapi::um::winnt::{GENERIC_READ, GENERIC_WRITE};
+
+bitflags! {
+    pub struct AgentFlags : u64 {
+        const CONERR = WINPTY_FLAG_CONERR;
+        const PLAIN_OUTPUT = WINPTY_FLAG_PLAIN_OUTPUT;
+        const COLOR_ESCAPES = WINPTY_FLAG_COLOR_ESCAPES;
+        const ALLOW_DESKTOP_CREATE = WINPTY_FLAG_ALLOW_CURPROC_DESKTOP_CREATION;
+    }
+}
+bitflags! {
+    pub struct SpawnFlags : u64 {
+        const AUTO_SHUTDOWN = WINPTY_SPAWN_FLAG_AUTO_SHUTDOWN;
+        const EXIT_AFTER_SHUTDOWN = WINPTY_SPAWN_FLAG_EXIT_AFTER_SHUTDOWN;
+    }
+}
+
+#[repr(u32)]
+enum MouseMode {
+    None = WINPTY_MOUSE_MODE_NONE,
+    Auto = WINPTY_MOUSE_MODE_AUTO,
+    Force = WINPTY_MOUSE_MODE_FORCE,
+}
+
+enum Timeout {
+    Infinite,
+    Milliseconds(DWORD),
+}
+
+struct WinPtyConfig {
+    config: *mut winpty_config_t,
+}
+
+fn wstr_to_osstr(wstr: LPCWSTR) -> Result<OsString, Error> {
+    ensure!(!wstr.is_null(), "LPCWSTR is null");
+    let slice = unsafe { std::slice::from_raw_parts(wstr, libc::wcslen(wstr)) };
+    Ok(OsString::from_wide(slice))
+}
+
+fn wstr_to_string(wstr: LPCWSTR) -> Result<String, Error> {
+    ensure!(!wstr.is_null(), "LPCWSTR is null");
+    let slice = unsafe { std::slice::from_raw_parts(wstr, libc::wcslen(wstr)) };
+    String::from_utf16(slice).map_err(|e| format_err!("String::from_utf16: {}", e))
+}
+
+fn check_err<T>(err: winpty_error_ptr_t, value: T) -> Result<T, Error> {
+    ensure!(!err.is_null(), "winpty error object is null");
+    unsafe {
+        let code = (WINPTY.winpty_error_code)(err);
+        if code == WINPTY_ERROR_SUCCESS {
+            return Ok(value);
+        }
+
+        let converted = wstr_to_string((WINPTY.winpty_error_msg)(err))?;
+        (WINPTY.winpty_error_free)(err);
+        bail!("winpty error code {}: {}", code, converted)
+    }
+}
+
+impl WinPtyConfig {
+    pub fn new(flags: AgentFlags) -> Result<Self, Error> {
+        let mut err: winpty_error_ptr_t = ptr::null_mut();
+        let config = unsafe { (WINPTY.winpty_config_new)(flags.bits(), &mut err) };
+        let config = check_err(err, config)?;
+        ensure!(
+            !config.is_null(),
+            "winpty_config_new returned nullptr but no error"
+        );
+        Ok(Self { config })
+    }
+
+    pub fn set_initial_size(&mut self, cols: c_int, rows: c_int) {
+        unsafe { (WINPTY.winpty_config_set_initial_size)(self.config, cols, rows) }
+    }
+
+    pub fn set_mouse_mode(&mut self, mode: MouseMode) {
+        unsafe { (WINPTY.winpty_config_set_mouse_mode)(self.config, mode as c_int) }
+    }
+
+    pub fn set_agent_timeout(&mut self, timeout: Timeout) {
+        let duration = match timeout {
+            Timeout::Infinite => INFINITE,
+            Timeout::Milliseconds(n) => n,
+        };
+        unsafe { (WINPTY.winpty_config_set_agent_timeout)(self.config, duration) }
+    }
+
+    pub fn open(&self) -> Result<WinPty, Error> {
+        let mut err: winpty_error_ptr_t = ptr::null_mut();
+        let pty = unsafe { (WINPTY.winpty_open)(self.config, &mut err) };
+        let pty = check_err(err, pty)?;
+        ensure!(!pty.is_null(), "winpty_open returned nullptr but no error");
+        Ok(WinPty { pty })
+    }
+}
+
+impl Drop for WinPtyConfig {
+    fn drop(&mut self) {
+        unsafe { (WINPTY.winpty_config_free)(self.config) }
+    }
+}
+
+pub struct WinPty {
+    pty: *mut winpty_t,
+}
+
+impl Drop for WinPty {
+    fn drop(&mut self) {
+        unsafe { (WINPTY.winpty_free)(self.pty) }
+    }
+}
+
+fn pipe_client(name: LPCWSTR) -> Result<OwnedHandle, Error> {
+    let handle = unsafe {
+        CreateFileW(
+            name,
+            GENERIC_READ | GENERIC_WRITE,
+            0,
+            ptr::null_mut(),
+            OPEN_EXISTING,
+            0,
+            ptr::null_mut(),
+        )
+    };
+    if handle == INVALID_HANDLE_VALUE {
+        let err = std::io::Error::last_os_error();
+        bail!("failed to open {:?}: {}", wstr_to_string(name), err);
+    } else {
+        Ok(OwnedHandle::new(handle))
+    }
+}
+
+impl WinPty {
+    pub fn agent_process(&self) -> HANDLE {
+        unsafe { (WINPTY.winpty_agent_process)(self.pty) }
+    }
+
+    pub fn conin(&self) -> Result<OwnedHandle, Error> {
+        pipe_client(unsafe { (WINPTY.winpty_conin_name)(self.pty) })
+    }
+
+    pub fn conout(&self) -> Result<OwnedHandle, Error> {
+        pipe_client(unsafe { (WINPTY.winpty_conout_name)(self.pty) })
+    }
+
+    pub fn conerr(&self) -> Result<OwnedHandle, Error> {
+        pipe_client(unsafe { (WINPTY.winpty_conerr_name)(self.pty) })
+    }
+
+    pub fn set_size(&mut self, rows: c_int, cols: c_int) -> Result<bool, Error> {
+        let mut err: winpty_error_ptr_t = ptr::null_mut();
+        let result = unsafe { (WINPTY.winpty_set_size)(self.pty, rows, cols, &mut err) };
+        Ok(result != 0)
+    }
+
+    pub fn spawn(&mut self, config: &SpawnConfig) -> Result<SpawnedProcess, Error> {
+        let mut err: winpty_error_ptr_t = ptr::null_mut();
+        let mut create_process_error: DWORD = 0;
+        let mut process_handle: HANDLE = ptr::null_mut();
+        let mut thread_handle: HANDLE = ptr::null_mut();
+
+        let result = unsafe {
+            (WINPTY.winpty_spawn)(
+                self.pty,
+                config.spawn_config,
+                &mut process_handle,
+                &mut thread_handle,
+                &mut create_process_error,
+                &mut err,
+            )
+        };
+        let thread_handle = OwnedHandle::new(thread_handle);
+        let process_handle = OwnedHandle::new(process_handle);
+        let result = check_err(err, result)?;
+        if result == 0 {
+            let err = std::io::Error::from_raw_os_error(create_process_error as _);
+            bail!("winpty_spawn failed: {}", err);
+        }
+        Ok(SpawnedProcess {
+            thread_handle,
+            process_handle,
+        })
+    }
+}
+
+pub struct SpawnedProcess {
+    process_handle: OwnedHandle,
+    thread_handle: OwnedHandle,
+}
+
+pub struct SpawnConfig {
+    spawn_config: *mut winpty_spawn_config_t,
+}
+
+/// Construct a null terminated wide string from an OsStr
+fn str_to_wide(s: &OsStr) -> Vec<u16> {
+    let mut wide: Vec<u16> = s.encode_wide().collect();
+    wide.push(0);
+    wide
+}
+
+fn str_ptr(s: &Option<Vec<u16>>) -> LPCWSTR {
+    match s {
+        None => ptr::null(),
+        Some(v) => v.as_ptr(),
+    }
+}
+
+impl SpawnConfig {
+    pub fn new(
+        flags: SpawnFlags,
+        appname: Option<&OsStr>,
+        cmdline: Option<&OsStr>,
+        cwd: Option<&OsStr>,
+        env: Option<&OsStr>,
+    ) -> Result<Self, Error> {
+        let appname = appname.map(str_to_wide);
+        let cmdline = cmdline.map(str_to_wide);
+        let cwd = cwd.map(str_to_wide);
+        let env = env.map(str_to_wide);
+
+        let mut err: winpty_error_ptr_t = ptr::null_mut();
+
+        let spawn_config = unsafe {
+            (WINPTY.winpty_spawn_config_new)(
+                flags.bits(),
+                str_ptr(&appname),
+                str_ptr(&cmdline),
+                str_ptr(&cwd),
+                str_ptr(&env),
+                &mut err,
+            )
+        };
+        let spawn_config = check_err(err, spawn_config)?;
+        ensure!(
+            !spawn_config.is_null(),
+            "winpty_spawn_config_new returned nullptr but no error"
+        );
+        Ok(Self { spawn_config })
+    }
+}
+
+impl Drop for SpawnConfig {
+    fn drop(&mut self) {
+        unsafe { (WINPTY.winpty_spawn_config_free)(self.spawn_config) }
+    }
+}