-
Notifications
You must be signed in to change notification settings - Fork 1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Improve heuristics preventing CPU/memory abuse (#515)
This addresses the following items: ==== Parse time of excessively deep nested or indented documents Parsing these documents is non-linear; limiting stack depth to 10,000 keeps parse times of pathological documents sub-second (~.25 seconds in benchmarks) ==== Alias node expansion limits The current limit allows 10,000% expansion, which is too permissive for large documents. Limiting to 10% expansion for larger documents allows callers to use input size as an effective way to limit resource usage. Continuing to allow larger expansion rates (up to the current 10,000% limit) for smaller documents does not unduly affect memory use. This change bounds decode operations from alias expansion to ~400,000 operations for small documents (worst-case ~100-150MB) or 10% of the input document for large documents, whichever is greater.
- Loading branch information
Showing
3 changed files
with
165 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,123 @@ | ||
package yaml_test | ||
|
||
import ( | ||
"strings" | ||
"testing" | ||
|
||
. "gopkg.in/check.v1" | ||
"gopkg.in/yaml.v2" | ||
) | ||
|
||
type testcase struct { | ||
name string | ||
data []byte | ||
error string | ||
} | ||
|
||
func testcases() []testcase { | ||
return []testcase{ | ||
{ | ||
name: "1000kb of maps with 100 aliases", | ||
data: []byte(`{a: &a [{a}` + strings.Repeat(`,{a}`, 1000*1024/4-100) + `], b: &b [*a` + strings.Repeat(`,*a`, 99) + `]}`), | ||
error: "yaml: document contains excessive aliasing", | ||
}, | ||
{ | ||
name: "1000kb of deeply nested slices", | ||
data: []byte(strings.Repeat(`[`, 1000*1024)), | ||
error: "yaml: exceeded max depth of 10000", | ||
}, | ||
{ | ||
name: "1000kb of deeply nested maps", | ||
data: []byte("x: " + strings.Repeat(`{`, 1000*1024)), | ||
error: "yaml: exceeded max depth of 10000", | ||
}, | ||
{ | ||
name: "1000kb of deeply nested indents", | ||
data: []byte(strings.Repeat(`- `, 1000*1024)), | ||
error: "yaml: exceeded max depth of 10000", | ||
}, | ||
{ | ||
name: "1000kb of 1000-indent lines", | ||
data: []byte(strings.Repeat(strings.Repeat(`- `, 1000)+"\n", 1024/2)), | ||
}, | ||
{name: "1kb of maps", data: []byte(`a: &a [{a}` + strings.Repeat(`,{a}`, 1*1024/4-1) + `]`)}, | ||
{name: "10kb of maps", data: []byte(`a: &a [{a}` + strings.Repeat(`,{a}`, 10*1024/4-1) + `]`)}, | ||
{name: "100kb of maps", data: []byte(`a: &a [{a}` + strings.Repeat(`,{a}`, 100*1024/4-1) + `]`)}, | ||
{name: "1000kb of maps", data: []byte(`a: &a [{a}` + strings.Repeat(`,{a}`, 1000*1024/4-1) + `]`)}, | ||
} | ||
} | ||
|
||
func (s *S) TestLimits(c *C) { | ||
if testing.Short() { | ||
return | ||
} | ||
for _, tc := range testcases() { | ||
var v interface{} | ||
err := yaml.Unmarshal(tc.data, &v) | ||
if len(tc.error) > 0 { | ||
c.Assert(err, ErrorMatches, tc.error, Commentf("testcase: %s", tc.name)) | ||
} else { | ||
c.Assert(err, IsNil, Commentf("testcase: %s", tc.name)) | ||
} | ||
} | ||
} | ||
|
||
func Benchmark1000KB100Aliases(b *testing.B) { | ||
benchmark(b, "1000kb of maps with 100 aliases") | ||
} | ||
func Benchmark1000KBDeeplyNestedSlices(b *testing.B) { | ||
benchmark(b, "1000kb of deeply nested slices") | ||
} | ||
func Benchmark1000KBDeeplyNestedMaps(b *testing.B) { | ||
benchmark(b, "1000kb of deeply nested maps") | ||
} | ||
func Benchmark1000KBDeeplyNestedIndents(b *testing.B) { | ||
benchmark(b, "1000kb of deeply nested indents") | ||
} | ||
func Benchmark1000KB1000IndentLines(b *testing.B) { | ||
benchmark(b, "1000kb of 1000-indent lines") | ||
} | ||
func Benchmark1KBMaps(b *testing.B) { | ||
benchmark(b, "1kb of maps") | ||
} | ||
func Benchmark10KBMaps(b *testing.B) { | ||
benchmark(b, "10kb of maps") | ||
} | ||
func Benchmark100KBMaps(b *testing.B) { | ||
benchmark(b, "100kb of maps") | ||
} | ||
func Benchmark1000KBMaps(b *testing.B) { | ||
benchmark(b, "1000kb of maps") | ||
} | ||
|
||
func benchmark(b *testing.B, name string) { | ||
var tc testcase | ||
for _, t := range testcases() { | ||
if t.name == name { | ||
tc = t | ||
break | ||
} | ||
} | ||
if tc.name != name { | ||
b.Errorf("testcase %q not found", name) | ||
return | ||
} | ||
|
||
b.ResetTimer() | ||
|
||
for i := 0; i < b.N; i++ { | ||
var v interface{} | ||
err := yaml.Unmarshal(tc.data, &v) | ||
if len(tc.error) > 0 { | ||
if err == nil { | ||
b.Errorf("expected error, got none") | ||
} else if err.Error() != tc.error { | ||
b.Errorf("expected error '%s', got '%s'", tc.error, err.Error()) | ||
} | ||
} else { | ||
if err != nil { | ||
b.Errorf("unexpected error: %v", err) | ||
} | ||
} | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters