0002-win32-Detect-Cygwin-MSYS-PTY.patch

From d7dfee733c29dae96984be373db934298d4ac53f Mon Sep 17 00:00:00 2001
From: "K.Takata" <kentkt@csc.jp>
Date: Thu, 1 Jun 2017 22:19:25 +0900
Subject: [PATCH 2/7] win32: Detect Cygwin/MSYS PTY

When running Win32 version of Ag on mintty or other Cygwin/MSYS terminal
emulators, Ag cannot detect Cygwin/MSYS pty and it causes some problems.
E.g.
https://github.com/ggreer/the_silver_searcher/issues/535#issuecomment-70944218
https://github.com/Alexpux/MSYS2-packages/pull/491

Import a check routine from https://github.com/k-takata/ptycheck .
---
 Makefile.am     |   2 +-
 Makefile.w32    |   5 +-
 configure.ac    |   1 +
 src/iscygpty.c  | 185 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 src/iscygpty.h  |  41 +++++++++++++
 src/options.c   |   5 +-
 src/print_w32.c |   2 +
 7 files changed, 236 insertions(+), 5 deletions(-)
 create mode 100644 src/iscygpty.c
 create mode 100644 src/iscygpty.h

diff --git a/Makefile.am b/Makefile.am
index 93aac08..29ad4bd 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,7 +1,7 @@
 ACLOCAL_AMFLAGS = ${ACLOCAL_FLAGS}
 
 bin_PROGRAMS = ag
-ag_SOURCES = src/ignore.c src/ignore.h src/log.c src/log.h src/options.c src/options.h src/print.c src/print_w32.c src/print.h src/scandir.c src/scandir.h src/search.c src/search.h src/lang.c src/lang.h src/util.c src/util.h src/decompress.c src/decompress.h src/uthash.h src/main.c
+ag_SOURCES = src/ignore.c src/ignore.h src/log.c src/log.h src/options.c src/options.h src/print.c src/print_w32.c src/print.h src/scandir.c src/scandir.h src/search.c src/search.h src/lang.c src/lang.h src/util.c src/util.h src/decompress.c src/decompress.h src/uthash.h src/main.c src/iscygpty.c src/iscygpty.h
 ag_LDADD = ${PCRE_LIBS} ${LZMA_LIBS} ${ZLIB_LIBS} $(PTHREAD_LIBS)
 
 dist_man_MANS = doc/ag.1
diff --git a/Makefile.w32 b/Makefile.w32
index 2794651..9688bba 100644
--- a/Makefile.w32
+++ b/Makefile.w32
@@ -15,10 +15,11 @@ SRCS = \
 	src/scandir.c \
 	src/search.c \
 	src/util.c \
-	src/print_w32.c
+	src/print_w32.c \
+	src/iscygpty.c
 OBJS = $(subst .c,.o,$(SRCS))
 
-CFLAGS = -O2 -Isrc/win32 -DPACKAGE_VERSION=\"$(VERSION)\"
+CFLAGS = -O2 -Isrc/win32 -DPACKAGE_VERSION=\"$(VERSION)\" -D_WIN32_WINNT=0x0600
 LIBS = -lz -lpthread -lpcre -llzma -lshlwapi
 TARGET = ag.exe
 
diff --git a/configure.ac b/configure.ac
index 3fa9c98..8808c69 100644
--- a/configure.ac
+++ b/configure.ac
@@ -36,6 +36,7 @@ LDFLAGS="$LDFLAGS"
 case $host in
 *mingw*)
     AC_CHECK_LIB(shlwapi, main,, AC_MSG_ERROR(libshlwapi missing))
+    CFLAGS="$CFLAGS -D_WIN32_WINNT=0x0600"
 esac
 
 LIBS="$PTHREAD_LIBS $LIBS"
diff --git a/src/iscygpty.c b/src/iscygpty.c
new file mode 100644
index 0000000..6a0d63e
--- /dev/null
+++ b/src/iscygpty.c
@@ -0,0 +1,185 @@
+/*
+ * iscygpty.c -- part of ptycheck
+ * https://github.com/k-takata/ptycheck
+ *
+ * Copyright (c) 2015-2017 K.Takata
+ *
+ * You can redistribute it and/or modify it under the terms of either
+ * the MIT license (as described below) or the Vim license.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifdef _WIN32
+
+#include <ctype.h>
+#include <io.h>
+#include <wchar.h>
+#include <windows.h>
+
+#ifdef USE_FILEEXTD
+/* VC 7.1 or earlier doesn't support SAL. */
+# if !defined(_MSC_VER) || (_MSC_VER < 1400)
+#  define __out
+#  define __in
+#  define __in_opt
+# endif
+/* Win32 FileID API Library:
+ * http://www.microsoft.com/en-us/download/details.aspx?id=22599
+ * Needed for WinXP. */
+# include <fileextd.h>
+#else /* USE_FILEEXTD */
+/* VC 8 or earlier. */
+# if defined(_MSC_VER) && (_MSC_VER < 1500)
+#  ifdef ENABLE_STUB_IMPL
+#   define STUB_IMPL
+#  else
+#   error "Win32 FileID API Library is required for VC2005 or earlier."
+#  endif
+# endif
+#endif /* USE_FILEEXTD */
+
+
+#include "iscygpty.h"
+
+//#define USE_DYNFILEID
+#ifdef USE_DYNFILEID
+typedef BOOL (WINAPI *pfnGetFileInformationByHandleEx)(
+		HANDLE                    hFile,
+		FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
+		LPVOID                    lpFileInformation,
+		DWORD                     dwBufferSize
+);
+static pfnGetFileInformationByHandleEx pGetFileInformationByHandleEx = NULL;
+
+# ifndef USE_FILEEXTD
+static BOOL WINAPI stub_GetFileInformationByHandleEx(
+		HANDLE                    hFile,
+		FILE_INFO_BY_HANDLE_CLASS FileInformationClass,
+		LPVOID                    lpFileInformation,
+		DWORD                     dwBufferSize
+		)
+{
+	return FALSE;
+}
+# endif
+
+static void setup_fileid_api(void)
+{
+	if (pGetFileInformationByHandleEx != NULL) {
+		return;
+	}
+	pGetFileInformationByHandleEx = (pfnGetFileInformationByHandleEx)
+		GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")),
+				"GetFileInformationByHandleEx");
+	if (pGetFileInformationByHandleEx == NULL) {
+# ifdef USE_FILEEXTD
+		pGetFileInformationByHandleEx = GetFileInformationByHandleEx;
+# else
+		pGetFileInformationByHandleEx = stub_GetFileInformationByHandleEx;
+# endif
+	}
+}
+#else
+# define pGetFileInformationByHandleEx	GetFileInformationByHandleEx
+# define setup_fileid_api()
+#endif
+
+
+#define is_wprefix(s, prefix) \
+	(wcsncmp((s), (prefix), sizeof(prefix) / sizeof(WCHAR) - 1) == 0)
+
+/* Check if the fd is a cygwin/msys's pty. */
+int is_cygpty(int fd)
+{
+#ifdef STUB_IMPL
+	return 0;
+#else
+	HANDLE h;
+	int size = sizeof(FILE_NAME_INFO) + sizeof(WCHAR) * MAX_PATH;
+	FILE_NAME_INFO *nameinfo;
+	WCHAR *p = NULL;
+
+	setup_fileid_api();
+
+	h = (HANDLE) _get_osfhandle(fd);
+	if (h == INVALID_HANDLE_VALUE) {
+		return 0;
+	}
+	/* Cygwin/msys's pty is a pipe. */
+	if (GetFileType(h) != FILE_TYPE_PIPE) {
+		return 0;
+	}
+	nameinfo = malloc(size);
+	if (nameinfo == NULL) {
+		return 0;
+	}
+	/* Check the name of the pipe:
+	 * '\{cygwin,msys}-XXXXXXXXXXXXXXXX-ptyN-{from,to}-master' */
+	if (pGetFileInformationByHandleEx(h, FileNameInfo, nameinfo, size)) {
+		nameinfo->FileName[nameinfo->FileNameLength / sizeof(WCHAR)] = L'\0';
+		p = nameinfo->FileName;
+		if (is_wprefix(p, L"\\cygwin-")) {		/* Cygwin */
+			p += 8;
+		} else if (is_wprefix(p, L"\\msys-")) {	/* MSYS and MSYS2 */
+			p += 6;
+		} else {
+			p = NULL;
+		}
+		if (p != NULL) {
+			while (*p && isxdigit(*p))	/* Skip 16-digit hexadecimal. */
+				++p;
+			if (is_wprefix(p, L"-pty")) {
+				p += 4;
+			} else {
+				p = NULL;
+			}
+		}
+		if (p != NULL) {
+			while (*p && isdigit(*p))	/* Skip pty number. */
+				++p;
+			if (is_wprefix(p, L"-from-master")) {
+				//p += 12;
+			} else if (is_wprefix(p, L"-to-master")) {
+				//p += 10;
+			} else {
+				p = NULL;
+			}
+		}
+	}
+	free(nameinfo);
+	return (p != NULL);
+#endif /* STUB_IMPL */
+}
+
+/* Check if at least one cygwin/msys pty is used. */
+int is_cygpty_used(void)
+{
+	int fd, ret = 0;
+
+	for (fd = 0; fd < 3; fd++) {
+		ret |= is_cygpty(fd);
+	}
+	return ret;
+}
+
+#endif /* _WIN32 */
+
+/* vim: set ts=4 sw=4: */
diff --git a/src/iscygpty.h b/src/iscygpty.h
new file mode 100644
index 0000000..82fd0af
--- /dev/null
+++ b/src/iscygpty.h
@@ -0,0 +1,41 @@
+/*
+ * iscygpty.h -- part of ptycheck
+ * https://github.com/k-takata/ptycheck
+ *
+ * Copyright (c) 2015-2017 K.Takata
+ *
+ * You can redistribute it and/or modify it under the terms of either
+ * the MIT license (as described below) or the Vim license.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _ISCYGPTY_H
+#define _ISCYGPTY_H
+
+#ifdef _WIN32
+int is_cygpty(int fd);
+int is_cygpty_used(void);
+#else
+#define is_cygpty(fd)		0
+#define is_cygpty_used()	0
+#endif
+
+#endif /* _ISCYGPTY_H */
diff --git a/src/options.c b/src/options.c
index b8da938..83152c5 100644
--- a/src/options.c
+++ b/src/options.c
@@ -10,6 +10,7 @@
 
 #include "config.h"
 #include "ignore.h"
+#include "iscygpty.h"
 #include "lang.h"
 #include "log.h"
 #include "options.h"
@@ -350,7 +351,7 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
 
     rv = fstat(fileno(stdin), &statbuf);
     if (rv == 0) {
-        if (S_ISFIFO(statbuf.st_mode) || S_ISREG(statbuf.st_mode)) {
+        if ((S_ISFIFO(statbuf.st_mode) || S_ISREG(statbuf.st_mode)) && !is_cygpty(fileno(stdin))) {
             opts.search_stream = 1;
         }
     }
@@ -359,7 +360,7 @@ void parse_options(int argc, char **argv, char **base_paths[], char **paths[]) {
         * turn off colors
         * print filenames on every line
      */
-    if (!isatty(fileno(stdout))) {
+    if (!isatty(fileno(stdout)) && !is_cygpty(fileno(stdout))) {
         opts.color = 0;
         group = 0;
 
diff --git a/src/print_w32.c b/src/print_w32.c
index a1fd387..f0a8de6 100644
--- a/src/print_w32.c
+++ b/src/print_w32.c
@@ -1,5 +1,6 @@
 #ifdef _WIN32
 
+#include "iscygpty.h"
 #include "print.h"
 #include <io.h>
 #include <stdarg.h>
@@ -47,6 +48,7 @@ int fprintf_w32(FILE *fp, const char *format, ...) {
     // the user just prefers ansi, do plain passthrough.
     BOOL passthrough =
         g_use_ansi ||
+        is_cygpty(fileno(fp)) ||
         !isatty(fileno(fp)) ||
         INVALID_HANDLE_VALUE == (stdo = (HANDLE)_get_osfhandle(fileno(fp))) ||
         !GetConsoleScreenBufferInfo(stdo, &csbi);
-- 
2.12.1