Skip to content

Commit

Permalink
interp: fix multi-line shell pattern match equality
Browse files Browse the repository at this point in the history 
commit be03afe added the multi-line
flag to the regexp generated for `==` and `=!` testing. However, this
flag is confusing as it allows for a regular expression to match one
line out of a multi-line input.

    (m) multi-line mode: ^ and $ match begin/end line in addition to
    begin/end text (default false)

This means we are now matching against just one line in a multi-line
string, rather than the entire text. Instead, to match Bash's behavior,
we add the `s` flag to the regexp:

    (s) let . match \n (default false)

With this a `*` in a shell pattern will match `\n`.

Glob patterns should always match newline characters, so add the `s`
flag by default when creating a pattern. Alternately we could add the
`s` flag only when the `.` meta-character is needed, but that seemed
more fragile. This behavior worked by accident with file globs because
rather than matching with `.*` we match with `[^/]*` which does match
`\n`.
  • Loading branch information
@lollipopman
lollipopman committed May 26, 2022
1 parent 169e304 commit 6f30262
Show file tree
Hide file tree
Showing 6 changed files with 94 additions and 62 deletions.
13 changes: 7 additions & 6 deletions expand/expand.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -845,21 +845,22 @@ func (cfg *Config) glob(base, pat string) ([]string, error) {

// If dir is not a directory, we keep the stack as-is and continue.
newMatches = newMatches[:0]
newMatches, _ = cfg.globDir(base, dir, rxGlobStar, wantDir, newMatches)
newMatches, _ = cfg.globDir(base, dir, rxGlobStar, false, wantDir, newMatches)
for i := len(newMatches) - 1; i >= 0; i-- {
stack = append(stack, newMatches[i])
}
}
continue
}
expr, err := pattern.Regexp(part, pattern.Filenames)
expr, err := pattern.Regexp(part, pattern.Filenames|pattern.EntireString)
if err != nil {
return nil, err
}
rx := regexp.MustCompile("^" + expr + "$")
rx := regexp.MustCompile(expr)
matchHidden := part[0] == byte('.')
var newMatches []string
for _, dir := range matches {
newMatches, err = cfg.globDir(base, dir, rx, wantDir, newMatches)
newMatches, err = cfg.globDir(base, dir, rx, matchHidden, wantDir, newMatches)
if err != nil {
return nil, err
}
Expand All @@ -869,7 +870,7 @@ func (cfg *Config) glob(base, pat string) ([]string, error) {
return matches, nil
}

func (cfg *Config) globDir(base, dir string, rx *regexp.Regexp, wantDir bool, matches []string) ([]string, error) {
func (cfg *Config) globDir(base, dir string, rx *regexp.Regexp, matchHidden bool, wantDir bool, matches []string) ([]string, error) {
fullDir := dir
if !filepath.IsAbs(dir) {
fullDir = filepath.Join(base, dir)
Expand All @@ -896,7 +897,7 @@ func (cfg *Config) globDir(base, dir string, rx *regexp.Regexp, wantDir bool, ma
// Not a symlink nor a directory.
continue
}
if !strings.HasPrefix(rx.String(), `^\.`) && name[0] == '.' {
if !matchHidden && name[0] == '.' {
continue
}
if rx.MatchString(name) {
Expand Down
22 changes: 22 additions & 0 deletions interp/interp_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1555,6 +1555,23 @@ var runTests = []runTest{
"[[ \"multiline\ntext\" == *text* ]] && echo x; [[ \"multiline\ntext\" == *multiline* ]] && echo y",
"x\ny\n",
},
// * should match a newline
{
"[[ \"multiline\ntext\" == multiline*text ]] && echo x",
"x\n",
},
{
"[[ \"multiline\ntext\" == text ]]",
"exit status 1",
},
{
`case $'a\nb' in a*b) echo match ;; esac`,
"match\n",
},
{
`a=$'a\nb'; echo "${a/a*b/sub}"`,
"sub\n",
},
{
"mkdir a; cd a; test -f b && echo x; >b; test -f b && echo y",
"y\n",
Expand Down Expand Up @@ -3214,6 +3231,11 @@ hello, world
`mapfile -t -d "" < <(printf "a\0b\n"); for x in "${MAPFILE[@]}"; do echo "$x"; done`,
"a\nb\n\n",
},
// Windows does not support having a `\n` in a filename
{
`> $'bar\nbaz'; echo bar*baz`,
"bar\nbaz\n",
},
}

var runTestsWindows = []runTest{
Expand Down
4 changes: 2 additions & 2 deletions interp/runner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -705,11 +705,11 @@ func (r *Runner) flattenAssign(as *syntax.Assign) []*syntax.Assign {
}

func match(pat, name string) bool {
expr, err := pattern.Regexp(pat, 0)
expr, err := pattern.Regexp(pat, pattern.EntireString)
if err != nil {
return false
}
rx := regexp.MustCompile("(?m)^" + expr + "$")
rx := regexp.MustCompile(expr)
return rx.MatchString(name)
}

Expand Down
2 changes: 1 addition & 1 deletion pattern/example_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ func ExampleRegexp() {
fmt.Println(rx.MatchString("foobarbaz"))
// Output:
// foo?bar*
// foo.bar.*
// (?s)foo.bar.*
// true
// false
}
Expand Down
17 changes: 13 additions & 4 deletions pattern/pattern.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,10 @@ func (e SyntaxError) Error() string { return e.msg }
func (e SyntaxError) Unwrap() error { return e.err }

const (
Shortest Mode = 1 << iota // prefer the shortest match.
Filenames // "*" and "?" don't match slashes; only "**" does
Braces // support "{a,b}" and "{1..4}"
Shortest Mode = 1 << iota // prefer the shortest match.
Filenames // "*" and "?" don't match slashes; only "**" does
Braces // support "{a,b}" and "{1..4}"
EntireString // match the entire string using ^$ delimiters
)

var numRange = regexp.MustCompile(`^([+-]?\d+)\.\.([+-]?\d+)}`)
Expand All @@ -59,11 +60,16 @@ noopLoop:
break noopLoop
}
}
if !any { // short-cut without a string copy
if !any && mode&EntireString == 0 { // short-cut without a string copy
return pat, nil
}
closingBraces := []int{}
var buf bytes.Buffer
// Enable matching `\n` with `.` as globs match `\n`
buf.WriteString("(?s)")
if mode&EntireString != 0 {
buf.WriteString("^")
}
writeLoop:
for i := 0; i < len(pat); i++ {
switch c := pat[i]; c {
Expand Down Expand Up @@ -228,6 +234,9 @@ writeLoop:
}
}
}
if mode&EntireString != 0 {
buf.WriteString("$")
}
return buf.String(), nil
}

Expand Down
98 changes: 49 additions & 49 deletions pattern/pattern_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,65 +18,65 @@ var translateTests = []struct {
{pat: ``, want: ``},
{pat: `foo`, want: `foo`},
{pat: `foóà中`, mode: Filenames | Braces, want: `foóà中`},
{pat: `.`, want: `\.`},
{pat: `foo*`, want: `foo.*`},
{pat: `foo*`, mode: Shortest, want: `foo.*?`},
{pat: `foo*`, mode: Shortest | Filenames, want: `foo[^/]*?`},
{pat: `*foo`, mode: Filenames, want: `[^/]*foo`},
{pat: `**`, want: `.*.*`},
{pat: `**`, mode: Filenames, want: `.*`},
{pat: `/**/foo`, want: `/.*.*/foo`},
{pat: `/**/foo`, mode: Filenames, want: `/(.*/|)foo`},
{pat: `/**/à`, mode: Filenames, want: `/(.*/|)à`},
{pat: `/**foo`, mode: Filenames, want: `/.*foo`},
{pat: `\*`, want: `\*`},
{pat: `.`, want: `(?s)\.`},
{pat: `foo*`, want: `(?s)foo.*`},
{pat: `foo*`, mode: Shortest, want: `(?s)foo.*?`},
{pat: `foo*`, mode: Shortest | Filenames, want: `(?s)foo[^/]*?`},
{pat: `*foo`, mode: Filenames, want: `(?s)[^/]*foo`},
{pat: `**`, want: `(?s).*.*`},
{pat: `**`, mode: Filenames, want: `(?s).*`},
{pat: `/**/foo`, want: `(?s)/.*.*/foo`},
{pat: `/**/foo`, mode: Filenames, want: `(?s)/(.*/|)foo`},
{pat: `/**/à`, mode: Filenames, want: `(?s)/(.*/|)à`},
{pat: `/**foo`, mode: Filenames, want: `(?s)/.*foo`},
{pat: `\*`, want: `(?s)\*`},
{pat: `\`, wantErr: true},
{pat: `?`, want: `.`},
{pat: `?`, mode: Filenames, want: `[^/]`},
{pat: `?à`, want: `.à`},
{pat: `\a`, want: `a`},
{pat: `(`, want: `\(`},
{pat: `a|b`, want: `a\|b`},
{pat: `x{3}`, want: `x\{3\}`},
{pat: `{3,4}`, want: `\{3,4\}`},
{pat: `{3,4}`, mode: Braces, want: `(?:3|4)`},
{pat: `{3,`, want: `\{3,`},
{pat: `{3,`, mode: Braces, want: `\{3,`},
{pat: `{3,{4}`, mode: Braces, want: `\{3,\{4\}`},
{pat: `{3,{4}}`, mode: Braces, want: `(?:3|\{4\})`},
{pat: `{3,{4,[56]}}`, mode: Braces, want: `(?:3|(?:4|[56]))`},
{pat: `{3..5}`, mode: Braces, want: `(?:3|4|5)`},
{pat: `{9..12}`, mode: Braces, want: `(?:9|10|11|12)`},
{pat: `[a]`, want: `[a]`},
{pat: `[abc]`, want: `[abc]`},
{pat: `[^bc]`, want: `[^bc]`},
{pat: `[!bc]`, want: `[^bc]`},
{pat: `[[]`, want: `[[]`},
{pat: `[\]]`, want: `[\]]`},
{pat: `[\]]`, mode: Filenames, want: `[\]]`},
{pat: `[]]`, want: `[]]`},
{pat: `[!]]`, want: `[^]]`},
{pat: `[^]]`, want: `[^]]`},
{pat: `[a/b]`, want: `[a/b]`},
{pat: `[a/b]`, mode: Filenames, want: `\[a/b\]`},
{pat: `?`, want: `(?s).`},
{pat: `?`, mode: Filenames, want: `(?s)[^/]`},
{pat: `?à`, want: `(?s).à`},
{pat: `\a`, want: `(?s)a`},
{pat: `(`, want: `(?s)\(`},
{pat: `a|b`, want: `(?s)a\|b`},
{pat: `x{3}`, want: `(?s)x\{3\}`},
{pat: `{3,4}`, want: `(?s)\{3,4\}`},
{pat: `{3,4}`, mode: Braces, want: `(?s)(?:3|4)`},
{pat: `{3,`, want: `(?s)\{3,`},
{pat: `{3,`, mode: Braces, want: `(?s)\{3,`},
{pat: `{3,{4}`, mode: Braces, want: `(?s)\{3,\{4\}`},
{pat: `{3,{4}}`, mode: Braces, want: `(?s)(?:3|\{4\})`},
{pat: `{3,{4,[56]}}`, mode: Braces, want: `(?s)(?:3|(?:4|[56]))`},
{pat: `{3..5}`, mode: Braces, want: `(?s)(?:3|4|5)`},
{pat: `{9..12}`, mode: Braces, want: `(?s)(?:9|10|11|12)`},
{pat: `[a]`, want: `(?s)[a]`},
{pat: `[abc]`, want: `(?s)[abc]`},
{pat: `[^bc]`, want: `(?s)[^bc]`},
{pat: `[!bc]`, want: `(?s)[^bc]`},
{pat: `[[]`, want: `(?s)[[]`},
{pat: `[\]]`, want: `(?s)[\]]`},
{pat: `[\]]`, mode: Filenames, want: `(?s)[\]]`},
{pat: `[]]`, want: `(?s)[]]`},
{pat: `[!]]`, want: `(?s)[^]]`},
{pat: `[^]]`, want: `(?s)[^]]`},
{pat: `[a/b]`, want: `(?s)[a/b]`},
{pat: `[a/b]`, mode: Filenames, want: `(?s)\[a/b\]`},
{pat: `[`, wantErr: true},
{pat: `[\`, wantErr: true},
{pat: `[^`, wantErr: true},
{pat: `[!`, wantErr: true},
{pat: `[!bc]`, want: `[^bc]`},
{pat: `[!bc]`, want: `(?s)[^bc]`},
{pat: `[]`, wantErr: true},
{pat: `[^]`, wantErr: true},
{pat: `[!]`, wantErr: true},
{pat: `[ab`, wantErr: true},
{pat: `[a-]`, want: `[a-]`},
{pat: `[a-]`, want: `(?s)[a-]`},
{pat: `[z-a]`, wantErr: true},
{pat: `[a-a]`, want: `[a-a]`},
{pat: `[aa]`, want: `[aa]`},
{pat: `[0-4A-Z]`, want: `[0-4A-Z]`},
{pat: `[-a]`, want: `[-a]`},
{pat: `[^-a]`, want: `[^-a]`},
{pat: `[a-]`, want: `[a-]`},
{pat: `[[:digit:]]`, want: `[[:digit:]]`},
{pat: `[a-a]`, want: `(?s)[a-a]`},
{pat: `[aa]`, want: `(?s)[aa]`},
{pat: `[0-4A-Z]`, want: `(?s)[0-4A-Z]`},
{pat: `[-a]`, want: `(?s)[-a]`},
{pat: `[^-a]`, want: `(?s)[^-a]`},
{pat: `[a-]`, want: `(?s)[a-]`},
{pat: `[[:digit:]]`, want: `(?s)[[:digit:]]`},
{pat: `[[:`, wantErr: true},
{pat: `[[:digit`, wantErr: true},
{pat: `[[:wrong:]]`, wantErr: true},
Expand Down

0 comments on commit 6f30262

Please sign in to comment.