From 3b1a54708d9f56517dc4214c9be9292d8d472325 Mon Sep 17 00:00:00 2001 From: brendan-matroid Date: Mon, 23 Mar 2020 17:36:22 -0400 Subject: [PATCH 01/16] Escape regex characters --- objurl/objurl.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/objurl/objurl.go b/objurl/objurl.go index 2b0497f0d..4405b652a 100644 --- a/objurl/objurl.go +++ b/objurl/objurl.go @@ -21,6 +21,9 @@ const ( // matchAllRe is the regex to match everything matchAllRe string = ".*" + + // regexCharacters need to be escaped to be interpreted literally + regexCharacters string = "\\^$.|?*+()[{" ) type objurlType int @@ -209,6 +212,9 @@ func (o *ObjectURL) setPrefixAndFilter() error { o.Prefix = o.Path[:loc] o.filter = o.Path[loc:] } + for _, char := range regexCharacters { + o.Prefix = strings.Replace(o.Prefix, string(char), "\\" + string(char), -1) + } filterRegex := matchAllRe if o.filter != "" { From 466b777d6e351b5a6dac28d491696aa9f9e735da Mon Sep 17 00:00:00 2001 From: brendan-matroid Date: Mon, 23 Mar 2020 17:49:54 -0400 Subject: [PATCH 02/16] Convert indentation to tabs --- objurl/objurl.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/objurl/objurl.go b/objurl/objurl.go index 4405b652a..68c1b1ca2 100644 --- a/objurl/objurl.go +++ b/objurl/objurl.go @@ -23,7 +23,7 @@ const ( matchAllRe string = ".*" // regexCharacters need to be escaped to be interpreted literally - regexCharacters string = "\\^$.|?*+()[{" + regexCharacters string = "\\^$.|?*+()[{" ) type objurlType int @@ -212,9 +212,9 @@ func (o *ObjectURL) setPrefixAndFilter() error { o.Prefix = o.Path[:loc] o.filter = o.Path[loc:] } - for _, char := range regexCharacters { - o.Prefix = strings.Replace(o.Prefix, string(char), "\\" + string(char), -1) - } + for _, char := range regexCharacters + o.Prefix = strings.Replace(o.Prefix, string(char), "\\" + string(char), -1) + } filterRegex := matchAllRe if o.filter != "" { From 8cee023829b76782b5a9ab59cd653302c0c023f7 Mon Sep 17 00:00:00 2001 From: brendan-matroid Date: Mon, 23 Mar 2020 17:50:52 -0400 Subject: [PATCH 03/16] Fix build --- objurl/objurl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/objurl/objurl.go b/objurl/objurl.go index 68c1b1ca2..d578707d1 100644 --- a/objurl/objurl.go +++ b/objurl/objurl.go @@ -212,7 +212,7 @@ func (o *ObjectURL) setPrefixAndFilter() error { o.Prefix = o.Path[:loc] o.filter = o.Path[loc:] } - for _, char := range regexCharacters + for _, char := range regexCharacters { o.Prefix = strings.Replace(o.Prefix, string(char), "\\" + string(char), -1) } From f94d54d2b5eefc3e15827bb6bf12ae261fd1651b Mon Sep 17 00:00:00 2001 From: brendan-matroid Date: Tue, 24 Mar 2020 00:26:56 +0000 Subject: [PATCH 04/16] Parse lines using shellquote --- command/cmd_run.go | 7 ++++++- go.mod | 1 + go.sum | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/command/cmd_run.go b/command/cmd_run.go index 411046a9e..036bd6c36 100644 --- a/command/cmd_run.go +++ b/command/cmd_run.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/urfave/cli/v2" + "github.com/kballard/go-shellquote" "github.com/peak/s5cmd/parallel" ) @@ -68,7 +69,11 @@ var RunCommand = &cli.Command{ continue } - fields := strings.Fields(line) + fields, err := shellquote.Split(line) + if err != nil { + return err + } + if len(fields) == 0 { continue } diff --git a/go.mod b/go.mod index 40fb17d8f..fa34e82e2 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334 github.com/johannesboyne/gofakes3 v0.0.0-20191228161223-9aee1c78a252 github.com/karrick/godirwalk v1.15.3 + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/kr/pretty v0.2.0 // indirect github.com/posener/complete v1.2.3 github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae diff --git a/go.sum b/go.sum index 9db4832b1..327ba0fca 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,8 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5i github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/karrick/godirwalk v1.15.3 h1:0a2pXOgtB16CqIqXTiT7+K9L73f74n/aNQUnH6Ortew= github.com/karrick/godirwalk v1.15.3/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= From 2d70bccf434159ca58d0ecde43f496b746a58b09 Mon Sep 17 00:00:00 2001 From: brendan-matroid Date: Tue, 24 Mar 2020 01:17:06 +0000 Subject: [PATCH 05/16] spaces -> tabs --- command/cmd_run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/cmd_run.go b/command/cmd_run.go index 036bd6c36..aa6138f5b 100644 --- a/command/cmd_run.go +++ b/command/cmd_run.go @@ -10,7 +10,7 @@ import ( "strings" "github.com/urfave/cli/v2" - "github.com/kballard/go-shellquote" + "github.com/kballard/go-shellquote" "github.com/peak/s5cmd/parallel" ) From 313326c021822632bee62c92abbb109b1c28fbc6 Mon Sep 17 00:00:00 2001 From: brendan-matroid Date: Tue, 24 Mar 2020 11:59:42 -0400 Subject: [PATCH 06/16] Vendor dependencies --- .../github.com/kballard/go-shellquote/LICENSE | 19 +++ .../github.com/kballard/go-shellquote/README | 36 ++++ .../github.com/kballard/go-shellquote/doc.go | 3 + .../kballard/go-shellquote/quote.go | 102 ++++++++++++ .../kballard/go-shellquote/unquote.go | 156 ++++++++++++++++++ vendor/modules.txt | 2 + 6 files changed, 318 insertions(+) create mode 100644 vendor/github.com/kballard/go-shellquote/LICENSE create mode 100644 vendor/github.com/kballard/go-shellquote/README create mode 100644 vendor/github.com/kballard/go-shellquote/doc.go create mode 100644 vendor/github.com/kballard/go-shellquote/quote.go create mode 100644 vendor/github.com/kballard/go-shellquote/unquote.go diff --git a/vendor/github.com/kballard/go-shellquote/LICENSE b/vendor/github.com/kballard/go-shellquote/LICENSE new file mode 100644 index 000000000..a6d77312e --- /dev/null +++ b/vendor/github.com/kballard/go-shellquote/LICENSE @@ -0,0 +1,19 @@ +Copyright (C) 2014 Kevin Ballard + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE +OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/kballard/go-shellquote/README b/vendor/github.com/kballard/go-shellquote/README new file mode 100644 index 000000000..4d34e87af --- /dev/null +++ b/vendor/github.com/kballard/go-shellquote/README @@ -0,0 +1,36 @@ +PACKAGE + +package shellquote + import "github.com/kballard/go-shellquote" + + Shellquote provides utilities for joining/splitting strings using sh's + word-splitting rules. + +VARIABLES + +var ( + UnterminatedSingleQuoteError = errors.New("Unterminated single-quoted string") + UnterminatedDoubleQuoteError = errors.New("Unterminated double-quoted string") + UnterminatedEscapeError = errors.New("Unterminated backslash-escape") +) + + +FUNCTIONS + +func Join(args ...string) string + Join quotes each argument and joins them with a space. If passed to + /bin/sh, the resulting string will be split back into the original + arguments. + +func Split(input string) (words []string, err error) + Split splits a string according to /bin/sh's word-splitting rules. It + supports backslash-escapes, single-quotes, and double-quotes. Notably it + does not support the $'' style of quoting. It also doesn't attempt to + perform any other sort of expansion, including brace expansion, shell + expansion, or pathname expansion. + + If the given input has an unterminated quoted string or ends in a + backslash-escape, one of UnterminatedSingleQuoteError, + UnterminatedDoubleQuoteError, or UnterminatedEscapeError is returned. + + diff --git a/vendor/github.com/kballard/go-shellquote/doc.go b/vendor/github.com/kballard/go-shellquote/doc.go new file mode 100644 index 000000000..9445fa4ad --- /dev/null +++ b/vendor/github.com/kballard/go-shellquote/doc.go @@ -0,0 +1,3 @@ +// Shellquote provides utilities for joining/splitting strings using sh's +// word-splitting rules. +package shellquote diff --git a/vendor/github.com/kballard/go-shellquote/quote.go b/vendor/github.com/kballard/go-shellquote/quote.go new file mode 100644 index 000000000..72a8cb38b --- /dev/null +++ b/vendor/github.com/kballard/go-shellquote/quote.go @@ -0,0 +1,102 @@ +package shellquote + +import ( + "bytes" + "strings" + "unicode/utf8" +) + +// Join quotes each argument and joins them with a space. +// If passed to /bin/sh, the resulting string will be split back into the +// original arguments. +func Join(args ...string) string { + var buf bytes.Buffer + for i, arg := range args { + if i != 0 { + buf.WriteByte(' ') + } + quote(arg, &buf) + } + return buf.String() +} + +const ( + specialChars = "\\'\"`${[|&;<>()*?!" + extraSpecialChars = " \t\n" + prefixChars = "~" +) + +func quote(word string, buf *bytes.Buffer) { + // We want to try to produce a "nice" output. As such, we will + // backslash-escape most characters, but if we encounter a space, or if we + // encounter an extra-special char (which doesn't work with + // backslash-escaping) we switch over to quoting the whole word. We do this + // with a space because it's typically easier for people to read multi-word + // arguments when quoted with a space rather than with ugly backslashes + // everywhere. + origLen := buf.Len() + + if len(word) == 0 { + // oops, no content + buf.WriteString("''") + return + } + + cur, prev := word, word + atStart := true + for len(cur) > 0 { + c, l := utf8.DecodeRuneInString(cur) + cur = cur[l:] + if strings.ContainsRune(specialChars, c) || (atStart && strings.ContainsRune(prefixChars, c)) { + // copy the non-special chars up to this point + if len(cur) < len(prev) { + buf.WriteString(prev[0 : len(prev)-len(cur)-l]) + } + buf.WriteByte('\\') + buf.WriteRune(c) + prev = cur + } else if strings.ContainsRune(extraSpecialChars, c) { + // start over in quote mode + buf.Truncate(origLen) + goto quote + } + atStart = false + } + if len(prev) > 0 { + buf.WriteString(prev) + } + return + +quote: + // quote mode + // Use single-quotes, but if we find a single-quote in the word, we need + // to terminate the string, emit an escaped quote, and start the string up + // again + inQuote := false + for len(word) > 0 { + i := strings.IndexRune(word, '\'') + if i == -1 { + break + } + if i > 0 { + if !inQuote { + buf.WriteByte('\'') + inQuote = true + } + buf.WriteString(word[0:i]) + } + word = word[i+1:] + if inQuote { + buf.WriteByte('\'') + inQuote = false + } + buf.WriteString("\\'") + } + if len(word) > 0 { + if !inQuote { + buf.WriteByte('\'') + } + buf.WriteString(word) + buf.WriteByte('\'') + } +} diff --git a/vendor/github.com/kballard/go-shellquote/unquote.go b/vendor/github.com/kballard/go-shellquote/unquote.go new file mode 100644 index 000000000..b1b13da93 --- /dev/null +++ b/vendor/github.com/kballard/go-shellquote/unquote.go @@ -0,0 +1,156 @@ +package shellquote + +import ( + "bytes" + "errors" + "strings" + "unicode/utf8" +) + +var ( + UnterminatedSingleQuoteError = errors.New("Unterminated single-quoted string") + UnterminatedDoubleQuoteError = errors.New("Unterminated double-quoted string") + UnterminatedEscapeError = errors.New("Unterminated backslash-escape") +) + +var ( + splitChars = " \n\t" + singleChar = '\'' + doubleChar = '"' + escapeChar = '\\' + doubleEscapeChars = "$`\"\n\\" +) + +// Split splits a string according to /bin/sh's word-splitting rules. It +// supports backslash-escapes, single-quotes, and double-quotes. Notably it does +// not support the $'' style of quoting. It also doesn't attempt to perform any +// other sort of expansion, including brace expansion, shell expansion, or +// pathname expansion. +// +// If the given input has an unterminated quoted string or ends in a +// backslash-escape, one of UnterminatedSingleQuoteError, +// UnterminatedDoubleQuoteError, or UnterminatedEscapeError is returned. +func Split(input string) (words []string, err error) { + var buf bytes.Buffer + words = make([]string, 0) + + for len(input) > 0 { + // skip any splitChars at the start + c, l := utf8.DecodeRuneInString(input) + if strings.ContainsRune(splitChars, c) { + input = input[l:] + continue + } else if c == escapeChar { + // Look ahead for escaped newline so we can skip over it + next := input[l:] + if len(next) == 0 { + err = UnterminatedEscapeError + return + } + c2, l2 := utf8.DecodeRuneInString(next) + if c2 == '\n' { + input = next[l2:] + continue + } + } + + var word string + word, input, err = splitWord(input, &buf) + if err != nil { + return + } + words = append(words, word) + } + return +} + +func splitWord(input string, buf *bytes.Buffer) (word string, remainder string, err error) { + buf.Reset() + +raw: + { + cur := input + for len(cur) > 0 { + c, l := utf8.DecodeRuneInString(cur) + cur = cur[l:] + if c == singleChar { + buf.WriteString(input[0 : len(input)-len(cur)-l]) + input = cur + goto single + } else if c == doubleChar { + buf.WriteString(input[0 : len(input)-len(cur)-l]) + input = cur + goto double + } else if c == escapeChar { + buf.WriteString(input[0 : len(input)-len(cur)-l]) + input = cur + goto escape + } else if strings.ContainsRune(splitChars, c) { + buf.WriteString(input[0 : len(input)-len(cur)-l]) + return buf.String(), cur, nil + } + } + if len(input) > 0 { + buf.WriteString(input) + input = "" + } + goto done + } + +escape: + { + if len(input) == 0 { + return "", "", UnterminatedEscapeError + } + c, l := utf8.DecodeRuneInString(input) + if c == '\n' { + // a backslash-escaped newline is elided from the output entirely + } else { + buf.WriteString(input[:l]) + } + input = input[l:] + } + goto raw + +single: + { + i := strings.IndexRune(input, singleChar) + if i == -1 { + return "", "", UnterminatedSingleQuoteError + } + buf.WriteString(input[0:i]) + input = input[i+1:] + goto raw + } + +double: + { + cur := input + for len(cur) > 0 { + c, l := utf8.DecodeRuneInString(cur) + cur = cur[l:] + if c == doubleChar { + buf.WriteString(input[0 : len(input)-len(cur)-l]) + input = cur + goto raw + } else if c == escapeChar { + // bash only supports certain escapes in double-quoted strings + c2, l2 := utf8.DecodeRuneInString(cur) + cur = cur[l2:] + if strings.ContainsRune(doubleEscapeChars, c2) { + buf.WriteString(input[0 : len(input)-len(cur)-l-l2]) + if c2 == '\n' { + // newline is special, skip the backslash entirely + } else { + buf.WriteRune(c2) + } + input = cur + } + } + } + return "", "", UnterminatedDoubleQuoteError + } + +done: + return buf.String(), input, nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 7cea80b79..04d35d625 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -72,6 +72,8 @@ github.com/johannesboyne/gofakes3/internal/goskipiter github.com/johannesboyne/gofakes3/internal/s3io # github.com/karrick/godirwalk v1.15.3 github.com/karrick/godirwalk +# github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 +github.com/kballard/go-shellquote # github.com/pkg/errors v0.9.1 github.com/pkg/errors # github.com/posener/complete v1.2.3 From 19750daccdcdfc089ca461955a03f25e44da0765 Mon Sep 17 00:00:00 2001 From: brendan-matroid Date: Tue, 24 Mar 2020 16:39:05 +0000 Subject: [PATCH 07/16] Use QuoteMeta --- objurl/objurl.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/objurl/objurl.go b/objurl/objurl.go index d578707d1..5583d3a71 100644 --- a/objurl/objurl.go +++ b/objurl/objurl.go @@ -21,9 +21,6 @@ const ( // matchAllRe is the regex to match everything matchAllRe string = ".*" - - // regexCharacters need to be escaped to be interpreted literally - regexCharacters string = "\\^$.|?*+()[{" ) type objurlType int @@ -212,9 +209,7 @@ func (o *ObjectURL) setPrefixAndFilter() error { o.Prefix = o.Path[:loc] o.filter = o.Path[loc:] } - for _, char := range regexCharacters { - o.Prefix = strings.Replace(o.Prefix, string(char), "\\" + string(char), -1) - } + o.Prefix = regexp.QuoteMeta(o.Prefix) filterRegex := matchAllRe if o.filter != "" { From b0887b6a1f373b21ec409d40099e8bb94a54677c Mon Sep 17 00:00:00 2001 From: brendan-matroid Date: Tue, 24 Mar 2020 16:49:58 +0000 Subject: [PATCH 08/16] Testcase for regex chars, special chars, spaces --- e2e/run_test.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/e2e/run_test.go b/e2e/run_test.go index 43a65466b..9f3b55d71 100644 --- a/e2e/run_test.go +++ b/e2e/run_test.go @@ -221,3 +221,34 @@ func TestRunWildcardCountGreaterEqualThanWorkerCount(t *testing.T) { 0: equals(""), }, strictLineCheck(true)) } + +func TestRunSpecialCharactersInPrefix(t *testing.T) { + t.Parallel() + + bucket := s3BucketFromTestName(t) + + s3client, s5cmd, cleanup := setup(t) + defer cleanup() + + createBucket(t, s3client, bucket) + putFile(t, s3client, bucket, "file.txt", "content") + + content := []string{ + "cp \"s3://" + bucket + "special-chars_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =/_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =image.jpg\" ./image.jpg", + } + file := fs.NewFile(t, "prefix", fs.WithContent(strings.Join(content, "\n"))) + defer file.Remove() + + cmd := s5cmd("run", file.Path()) + cmd.Timeout = time.Second + result := icmd.RunCmd(cmd) + result.Assert(t, icmd.Success) + + assertLines(t, result.Stdout(), map[int]compareFunc{ + 0: equals("cp s3://" + bucket + "/special-chars_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =/_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =image.jpg"), + }, sortInput(true)) + + assertLines(t, result.Stderr(), map[int]compareFunc{ + 0: equals(""), + }, strictLineCheck(true)) +} \ No newline at end of file From 78de7628d9afc99a902780962babd2a59769f0ba Mon Sep 17 00:00:00 2001 From: brendan-matroid Date: Sun, 12 Apr 2020 15:45:39 -0400 Subject: [PATCH 09/16] Don't overwrite o.Prefix --- objurl/objurl.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/objurl/objurl.go b/objurl/objurl.go index 5583d3a71..790a909a8 100644 --- a/objurl/objurl.go +++ b/objurl/objurl.go @@ -209,7 +209,6 @@ func (o *ObjectURL) setPrefixAndFilter() error { o.Prefix = o.Path[:loc] o.filter = o.Path[loc:] } - o.Prefix = regexp.QuoteMeta(o.Prefix) filterRegex := matchAllRe if o.filter != "" { @@ -217,7 +216,7 @@ func (o *ObjectURL) setPrefixAndFilter() error { filterRegex = strings.Replace(filterRegex, "\\?", ".", -1) filterRegex = strings.Replace(filterRegex, "\\*", ".*?", -1) } - filterRegex = o.Prefix + filterRegex + filterRegex = regexp.QuoteMeta(o.Prefix) + filterRegex r, err := regexp.Compile("^" + filterRegex + "$") if err != nil { return err From 9fe3e41355bbfab6e1045ce6224a52a746a36043 Mon Sep 17 00:00:00 2001 From: brendan-matroid Date: Sun, 12 Apr 2020 15:46:17 -0400 Subject: [PATCH 10/16] Add empty line 0, use backticks --- e2e/run_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/e2e/run_test.go b/e2e/run_test.go index 9f3b55d71..2f7b725cd 100644 --- a/e2e/run_test.go +++ b/e2e/run_test.go @@ -234,7 +234,7 @@ func TestRunSpecialCharactersInPrefix(t *testing.T) { putFile(t, s3client, bucket, "file.txt", "content") content := []string{ - "cp \"s3://" + bucket + "special-chars_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =/_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =image.jpg\" ./image.jpg", + `cp "s3://` + bucket + `/special-chars_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =/_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =image.jpg" ./image.jpg`, } file := fs.NewFile(t, "prefix", fs.WithContent(strings.Join(content, "\n"))) defer file.Remove() @@ -245,10 +245,11 @@ func TestRunSpecialCharactersInPrefix(t *testing.T) { result.Assert(t, icmd.Success) assertLines(t, result.Stdout(), map[int]compareFunc{ - 0: equals("cp s3://" + bucket + "/special-chars_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =/_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =image.jpg"), + 0: equals(""), + 1: equals(`cp s3://` + bucket + `/special-chars_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =/_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =image.jpg`), }, sortInput(true)) assertLines(t, result.Stderr(), map[int]compareFunc{ 0: equals(""), }, strictLineCheck(true)) -} \ No newline at end of file +} From 510f1540e84c166024c9887d37fceefef59f21a7 Mon Sep 17 00:00:00 2001 From: brendan-matroid Date: Sun, 12 Apr 2020 15:46:43 -0400 Subject: [PATCH 11/16] Don't Sprintf string if it has no arguments to allow for strings containing literal % --- e2e/util_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/e2e/util_test.go b/e2e/util_test.go index a9fc77cdf..35c41f456 100644 --- a/e2e/util_test.go +++ b/e2e/util_test.go @@ -414,7 +414,12 @@ func isJSON(str string) bool { } func equals(format string, args ...interface{}) compareFunc { - expected := fmt.Sprintf(format, args...) + expected := "" + if len(args) > 0 { + expected = fmt.Sprintf(format, args...) + } else { + expected = format + } return func(actual string) error { if expected == actual { return nil From 22dd92e832e60dd3de9c62dac210308bd294a396 Mon Sep 17 00:00:00 2001 From: brendan-matroid Date: Sun, 12 Apr 2020 16:11:49 -0400 Subject: [PATCH 12/16] Make testcase consistent with other cases --- e2e/run_test.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/e2e/run_test.go b/e2e/run_test.go index 866dccc7f..5926f4a8a 100644 --- a/e2e/run_test.go +++ b/e2e/run_test.go @@ -225,11 +225,8 @@ func TestRunSpecialCharactersInPrefix(t *testing.T) { result.Assert(t, icmd.Success) assertLines(t, result.Stdout(), map[int]compareFunc{ - 0: equals(""), - 1: equals(`cp s3://` + bucket + `/special-chars_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =/_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =image.jpg`), + 0: equals(`cp s3://` + bucket + `/special-chars_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =/_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =image.jpg`), }, sortInput(true)) - assertLines(t, result.Stderr(), map[int]compareFunc{ - 0: equals(""), - }, strictLineCheck(true)) + assertLines(t, result.Stderr(), map[int]compareFunc{}) } From 9291a9abe34060c562fdd2854e3d285db8c1537a Mon Sep 17 00:00:00 2001 From: brendan-matroid Date: Sun, 12 Apr 2020 16:17:18 -0400 Subject: [PATCH 13/16] Fix filename --- e2e/run_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/run_test.go b/e2e/run_test.go index 5926f4a8a..d31191d34 100644 --- a/e2e/run_test.go +++ b/e2e/run_test.go @@ -211,7 +211,7 @@ func TestRunSpecialCharactersInPrefix(t *testing.T) { defer cleanup() createBucket(t, s3client, bucket) - putFile(t, s3client, bucket, "file.txt", "content") + putFile(t, s3client, bucket, `special-chars_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =/_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =image.jpg`, "content") content := []string{ `cp "s3://` + bucket + `/special-chars_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =/_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =image.jpg" ./image.jpg`, From 0821d92b0b852b60803cf6c8dd20365352adcb0a Mon Sep 17 00:00:00 2001 From: brendan-matroid Date: Sun, 12 Apr 2020 16:22:36 -0400 Subject: [PATCH 14/16] Fix expected output --- e2e/run_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/run_test.go b/e2e/run_test.go index d31191d34..21cf7ee35 100644 --- a/e2e/run_test.go +++ b/e2e/run_test.go @@ -225,7 +225,7 @@ func TestRunSpecialCharactersInPrefix(t *testing.T) { result.Assert(t, icmd.Success) assertLines(t, result.Stdout(), map[int]compareFunc{ - 0: equals(`cp s3://` + bucket + `/special-chars_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =/_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =image.jpg`), + 0: equals(`cp s3://` + bucket + `/special-chars_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =/_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =image.jpg ./image.jpg`), }, sortInput(true)) assertLines(t, result.Stderr(), map[int]compareFunc{}) From 61d38e84ddc0f5ae6743390a1e44af18f9401f15 Mon Sep 17 00:00:00 2001 From: brendan-matroid Date: Sun, 12 Apr 2020 20:21:00 -0400 Subject: [PATCH 15/16] De-duplicate strings, use string formatting --- e2e/run_test.go | 8 +++++--- e2e/util_test.go | 7 +------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/e2e/run_test.go b/e2e/run_test.go index 21cf7ee35..33e432ee0 100644 --- a/e2e/run_test.go +++ b/e2e/run_test.go @@ -211,10 +211,12 @@ func TestRunSpecialCharactersInPrefix(t *testing.T) { defer cleanup() createBucket(t, s3client, bucket) - putFile(t, s3client, bucket, `special-chars_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =/_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =image.jpg`, "content") + sourceFileName := `special-chars_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =/_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =image.jpg` + targetFilePath := `./image.jpg` + putFile(t, s3client, bucket, sourceFileName, "content") content := []string{ - `cp "s3://` + bucket + `/special-chars_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =/_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =image.jpg" ./image.jpg`, + `cp "s3://` + bucket + `/` + sourceFileName + `" ` + targetFilePath, } file := fs.NewFile(t, "prefix", fs.WithContent(strings.Join(content, "\n"))) defer file.Remove() @@ -225,7 +227,7 @@ func TestRunSpecialCharactersInPrefix(t *testing.T) { result.Assert(t, icmd.Success) assertLines(t, result.Stdout(), map[int]compareFunc{ - 0: equals(`cp s3://` + bucket + `/special-chars_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =/_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =image.jpg ./image.jpg`), + 0: equals(`cp s3://%v/%v %v`, bucket, sourceFileName, targetFilePath), }, sortInput(true)) assertLines(t, result.Stderr(), map[int]compareFunc{}) diff --git a/e2e/util_test.go b/e2e/util_test.go index 9317e5525..7d6692edb 100644 --- a/e2e/util_test.go +++ b/e2e/util_test.go @@ -473,12 +473,7 @@ func isJSON(str string) bool { } func equals(format string, args ...interface{}) compareFunc { - expected := "" - if len(args) > 0 { - expected = fmt.Sprintf(format, args...) - } else { - expected = format - } + expected := fmt.Sprintf(format, args...) return func(actual string) error { if expected == actual { return nil From 96df1e05df5166663efadb23ac94de4f2ef8946c Mon Sep 17 00:00:00 2001 From: brendan-matroid Date: Sun, 12 Apr 2020 20:22:41 -0400 Subject: [PATCH 16/16] Move string declaration to top of function --- e2e/run_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/run_test.go b/e2e/run_test.go index 33e432ee0..b6f3699c7 100644 --- a/e2e/run_test.go +++ b/e2e/run_test.go @@ -206,13 +206,13 @@ func TestRunSpecialCharactersInPrefix(t *testing.T) { t.Parallel() bucket := s3BucketFromTestName(t) + sourceFileName := `special-chars_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =/_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =image.jpg` + targetFilePath := `./image.jpg` s3client, s5cmd, cleanup := setup(t) defer cleanup() createBucket(t, s3client, bucket) - sourceFileName := `special-chars_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =/_!@#$%^&_()_+{[_%5Cäè| __;'_,_._-中文 =image.jpg` - targetFilePath := `./image.jpg` putFile(t, s3client, bucket, sourceFileName, "content") content := []string{