From 8ff6d988918d1a7efb9bc613319ebc9b53b373dc Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Mon, 30 Apr 2018 16:30:07 -0500 Subject: [PATCH 1/2] Added an implementation of buildargv to internal/objabi/flag.go and modified Flagparse to use it when a response file is detected in os.Args. The buildargv function is pretty much copied from gcc's libiberty/argv.c --- src/cmd/internal/objabi/flag.go | 133 ++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/src/cmd/internal/objabi/flag.go b/src/cmd/internal/objabi/flag.go index 1bd4bc9063ae15..1861b899721e13 100644 --- a/src/cmd/internal/objabi/flag.go +++ b/src/cmd/internal/objabi/flag.go @@ -7,6 +7,8 @@ package objabi import ( "flag" "fmt" + "io/ioutil" + "log" "os" "strconv" "strings" @@ -27,8 +29,139 @@ func Flagprint(fd int) { flag.PrintDefaults() } +// This is a near exact copy of gcc's libiberty/argv.c buildargv +func buildargv(data []byte) []string { + var result []string + + // current index into data + var di int + + // string parsing states + var squote, dquote, bsquote bool + + // simple check if a char is whitespace + ISSPACE := func(ch byte) bool { + return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\v' || ch == '\f' || ch == '\r' + } + + // simple consume_whitespace implementation + consume_whitespace := func(idx int, data []byte) int { + for i := idx; i < len(data); i++ { + if !ISSPACE(data[i]) { + return i + } + } + return len(data) + } + + // consume initial whitespace + di = consume_whitespace(0, data) + if di >= len(data) { + return result + } + + // argument loop + for di < len(data) { + var arg string + + // scan each individual argument + arg = "" + for di < len(data) { + if ISSPACE(data[di]) && !squote && !dquote && !bsquote { + break + } + + // backslash + if bsquote { + bsquote = false + arg += string(data[di]) + } else if data[di] == '\\' { + bsquote = true + + // single-quote + } else if squote { + if data[di] == '\'' { + squote = false + } else { + arg += string(data[di]) + } + + // double-quote + } else if dquote { + if data[di] == '"' { + dquote = false + } else { + arg += string(data[di]) + } + + // state entries + } else { + if data[di] == '\'' { + squote = true + } else if data[di] == '"' { + dquote = true + } else { + arg += string(data[di]) + } + } + + // process the next byte + di += 1 + } + + // add the current arg to the results + result = append(result, arg) + di = consume_whitespace(di, data) + } + + // ...and we're done! + return result +} + func Flagparse(usage func()) { flag.Usage = usage + + // Expand any response files that were specified at the commandline. Anything + // that is not a response file (file not found, zero-length arg, etc) gets + // blindly added to the arg as we assume that the user knows what they're doing. + + // FIXME: I think response files are recursive, so if that's true then this + // code should be refactored to support recursive response files. Probably + // with a channel or something. + var args []string + for _, arg := range os.Args { + // Check that response file prefix doesn't exist or that arg is zero-length + if !strings.HasPrefix(arg, "@") || len(arg) < 1 { + args = append(args, arg) + continue + } + + // Check to see if the @-prefixed file is non-existent + file, err := os.Open(arg[1:]) + if os.IsNotExist(err) { + log.Printf("Unable to open response file (%s): %#v\n", arg[1:], err) + args = append(args, arg) + continue + } + + // Okay, so now we have a file with args. So expand the file contents + contents, err := ioutil.ReadAll(file) + if err != nil { + log.Fatalf("Unable to read contents of response file (%s): %#v", arg[1:], err) + } + + // Now we can add each arg from the file + for _, row := range buildargv(contents) { + args = append(args, row) + } + + // We're done. Close it and move on + if err := file.Close(); err != nil { + log.Fatalf("Unable to close response file (%s): %#v", arg[1:], err) + } + } + + os.Args = args flag.Parse() } From 8d90e4f44f2f7abbfa9c31794d2a42662a525f1d Mon Sep 17 00:00:00 2001 From: Ali Rizvi-Santiago Date: Mon, 30 Apr 2018 21:54:59 -0500 Subject: [PATCH 2/2] Added a function to src/cmd/go/internal/work/exec.go that copies arguments into a response file if the total length of all the args is larger than the limitation imposed by windows. Also Modified runOut to use this when executing a sub-command. Closes issue #18468 --- src/cmd/go/internal/work/exec.go | 103 ++++++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 1 deletion(-) diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index a50c996041262e..28ba9f7c093bc1 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -13,6 +13,7 @@ import ( "fmt" "io" "io/ioutil" + "bufio" "log" "os" "os/exec" @@ -1508,7 +1509,26 @@ func (b *Builder) runOut(dir string, desc string, env []string, cmdargs ...inter } var buf bytes.Buffer - cmd := exec.Command(cmdline[0], cmdline[1:]...) + + // Execute the requested cmdline. If we're on windows and the size of the + // cmdline is larger than the windows limitation, then write the arguments + // into a response file and use that. + response, cmd := buildLongCommand(cmdline) + + // if a response file was returned, then make sure to remove it after usage + if response != nil { + defer func(file *os.File) { + filename := file.Name() + if err := response.Close(); err != nil { + log.Fatalf("Unable to close response file (%s): %#v", filename, err) + } + if err := os.Remove(filename); err != nil { + log.Fatalf("Unable to remove response file (%s): %#v", filename, err) + } + }(response) + } + + // Continue populating the *exec.Cmd structure cmd.Stdout = &buf cmd.Stderr = &buf cmd.Dir = dir @@ -2428,3 +2448,84 @@ func mkAbsFiles(dir string, files []string) []string { } return abs } + +// Escape all the specified chars in the input string using a backslash. +func escapeString(input, characters string) string { + var result string + const escape = `\` + + // Iterate through the input string looking for any chars that match + for _, char := range input { + // If a specified char was found (or the escape char), then prefix it. + if strings.Contains(characters + escape, string(char)) { + result += escape + } + + // Now we can append the next character + result += string(char) + } + + // ...and now we should be good to go! + return result +} + +// buildLongCommand returns a response os.File and an os/exec.Cmd if we're on windows +// and the cmdline is larger than the limitation of UNICODE_STRING (0x8000) +// +// See Issue 18468. +func buildLongCommand(cmdline []string) (*os.File, *exec.Cmd) { + const MAX_UNICODE_STRING = 0x7fff + const IFS = "\n" + + const response_file_prefix = "resp-" + + // Sum up the total number of bytes occupied by the commandline + var argLen int + for _, arg := range cmdline { + argLen += len(arg) + argLen += len(" ") // command line arg separator + } + + // Check if we're not running windows, if there's no commandline arguments, + // or if there's no need to use a response file because we're under the limit. + if runtime.GOOS != "windows" || len(cmdline) <= 1 || argLen < MAX_UNICODE_STRING { + //log.Printf("Executing command (%s) normally (%d < %d).", cmdline[0], argLen, MAX_UNICODE_STRING) + return nil, exec.Command(cmdline[0], cmdline[1:]...) + } + //log.Printf("Executing command (%s) with response file (%d >= %d).", cmdline[0], argLen, MAX_UNICODE_STRING) + + // Create a temporary response file containing the old commandline arguments + file, err := ioutil.TempFile("", response_file_prefix) + if err != nil { + log.Fatalf("Unable to open up a temporary file to insert response parameters: %#v", err) + } + + // Populate temporary file with contents of cmdline[1:] + f := bufio.NewWriter(file) + for ai, arg := range cmdline[1:] { + var row string + + // FIXME: Figure out what the proper way to quote args in a response file are.. + + // If it's not already quoted, then double-quote and escape it + if !strings.HasPrefix(arg, `"`) && !strings.HasSuffix(arg, `"`) { + row = fmt.Sprintf(`"%s"`, escapeString(arg, `\"`)) + + // Otherwise we can just add it as-is since the user has already quoted it + } else { + row = arg + } + + // Write it to the response file separated by IFS + if _, err := fmt.Fprintf(f, `%s%s`, row, IFS); err != nil { + log.Fatalf("Unable to write cmdline[%d] to response file (%s): %#v", ai, file.Name(), err) + } + } + + // Flush everything written to the response file + if err := f.Flush(); err != nil { + log.Fatalf("Unable to flush output to response file (%s): %#v", file.Name(), err) + } + + return file, exec.Command(cmdline[0], "@" + file.Name()) +}