From 34ba868798afbc4aee7a6c7c9f5a9ebd3ac1b47e Mon Sep 17 00:00:00 2001
From: Mustafa M <mus-m@outlook.com>
Date: Tue, 21 Jul 2020 12:31:07 -0400
Subject: [PATCH] Reset REPL console mode to permit safely allowing tty, vt
 sequences on Windows   (#36598)

On Windows, when launching external processes, we cannot control what assumption they make on the
console mode. We thus forcibly reset the console mode at the start of the prompt to ensure they do
not leave the console mode in a corrupt state.
---
 src/support/libsupportinit.c |  4 +---
 stdlib/REPL/src/LineEdit.jl  | 31 +++++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/src/support/libsupportinit.c b/src/support/libsupportinit.c
index fe14d52711f66..1dbce675be34a 100644
--- a/src/support/libsupportinit.c
+++ b/src/support/libsupportinit.c
@@ -12,9 +12,7 @@ static int isInitialized = 0;
 void libsupport_init(void)
 {
     if (!isInitialized) {
-#ifdef _OS_WINDOWS_
-        SetConsoleCP(1252); // ANSI Latin1; Western European (Windows)
-#endif
+
         setlocale(LC_ALL, ""); // set to user locale
         setlocale(LC_NUMERIC, "C"); // use locale-independent numeric formats
 
diff --git a/stdlib/REPL/src/LineEdit.jl b/stdlib/REPL/src/LineEdit.jl
index 9b02815441bff..94115ab640eed 100644
--- a/stdlib/REPL/src/LineEdit.jl
+++ b/stdlib/REPL/src/LineEdit.jl
@@ -1272,8 +1272,39 @@ function write_prompt(terminal, p::Prompt)
     return width
 end
 
+# On Windows, when launching external processes, we cannot control what assumption they make on the
+# console mode. We thus forcibly reset the console mode at the start of the prompt to ensure they do
+# not leave the console mode in a corrupt state.
+# FIXME: remove when pseudo-tty are implemented for child processes
+if Sys.iswindows()
+function _console_mode()
+    hOutput = ccall(:GetStdHandle, stdcall, Ptr{Cvoid}, (UInt32,), -11 % UInt32) # STD_OUTPUT_HANDLE
+    dwMode = Ref{UInt32}()
+    ccall(:GetConsoleMode, stdcall, Int32, (Ref{Cvoid}, Ref{UInt32}), hOutput, dwMode)
+    return dwMode[]
+end
+const default_console_mode_ref = Ref{UInt32}()
+const default_console_mode_assigned = Ref(false)
+function get_default_console_mode()
+    if default_console_mode_assigned[] == false
+        default_console_mode_assigned[] = true
+        default_console_mode_ref[] = _console_mode()
+    end
+    return default_console_mode_ref[]
+end
+function _reset_console_mode()
+    mode = _console_mode()
+    if mode !== get_default_console_mode()
+        hOutput = ccall(:GetStdHandle, stdcall, Ptr{Cvoid}, (UInt32,), -11 % UInt32) # STD_OUTPUT_HANDLE
+        ccall(:SetConsoleMode, stdcall, Int32, (Ptr{Cvoid}, UInt32), hOutput, default_console_mode_ref[])
+    end
+    nothing
+end
+end
+
 # returns the width of the written prompt
 function write_prompt(terminal, s::Union{AbstractString,Function})
+    @static Sys.iswindows() && _reset_console_mode()
     promptstr = prompt_string(s)
     write(terminal, promptstr)
     return textwidth(promptstr)