From b5b49edfd7a7fdaa5a83045b91f86557ab3ab370 Mon Sep 17 00:00:00 2001 From: Abhinav Gupta Date: Sat, 21 Oct 2023 12:40:38 -0700 Subject: [PATCH 1/8] stack: Parse all functions Adds support to the stack parser for reading the full list of functions for a stack trace. This includes the function that created the stack trace; it's the bottom of the stack. We don't maintain the order of the functions since that's not something we need at this time. The functions are all placed in a set. --- internal/stack/stacks.go | 75 ++++++++++++++++++++++++++++++----- internal/stack/stacks_test.go | 70 +++++++++++++++++++++++++++++--- 2 files changed, 128 insertions(+), 17 deletions(-) diff --git a/internal/stack/stacks.go b/internal/stack/stacks.go index 5c4b9c4..a7411ec 100644 --- a/internal/stack/stacks.go +++ b/internal/stack/stacks.go @@ -34,10 +34,15 @@ const _defaultBufferSize = 64 * 1024 // 64 KiB // Stack represents a single Goroutine's stack. type Stack struct { - id int - state string + id int + state string // e.g. 'running', 'chan receive' + + // The first function on the stack. firstFunction string + // A set of all functions in the stack, + allFunctions map[string]struct{} + // Full, raw stack trace. fullStack string } @@ -62,6 +67,13 @@ func (s Stack) FirstFunction() string { return s.firstFunction } +// HasFunction reports whether the stack has the given function +// anywhere in it. +func (s Stack) HasFunction(name string) bool { + _, ok := s.allFunctions[name] + return ok +} + func (s Stack) String() string { return fmt.Sprintf( "Goroutine %v in state %v, with %v on top of the stack:\n%s", @@ -126,8 +138,12 @@ func (p *stackParser) parseStack(line string) (Stack, error) { firstFunction string fullStack bytes.Buffer ) + funcs := make(map[string]struct{}) for p.scan.Scan() { line := p.scan.Text() + if len(line) == 0 { + continue + } if strings.HasPrefix(line, "goroutine ") { // If we see the goroutine header, @@ -140,12 +156,29 @@ func (p *stackParser) parseStack(line string) (Stack, error) { fullStack.WriteString(line) fullStack.WriteByte('\n') // scanner trims the newline - // The first line after the header is the top of the stack. + funcName, err := parseFuncName(line) + if err != nil { + return Stack{}, fmt.Errorf("parse function: %w", err) + } + funcs[funcName] = struct{}{} if firstFunction == "" { - firstFunction, err = parseFirstFunc(line) - if err != nil { - return Stack{}, fmt.Errorf("extract function: %w", err) + firstFunction = funcName + } + + // The function name is usually followed by a line in the form: + // + // example.com/path/to/package/file.go:123 +0x123 + // + // We don't care about the position, so we'll skip it, + // but only if it matches the expected format. + if p.scan.Scan() { + bs := p.scan.Bytes() + if len(bs) > 0 && bs[0] == '\t' { + continue } + + // Put it back if it doesn't match. + p.scan.Unscan() } } @@ -153,6 +186,7 @@ func (p *stackParser) parseStack(line string) (Stack, error) { id: id, state: state, firstFunction: firstFunction, + allFunctions: funcs, fullStack: fullStack.String(), }, nil } @@ -176,12 +210,31 @@ func getStackBuffer(all bool) []byte { } } -func parseFirstFunc(line string) (string, error) { - line = strings.TrimSpace(line) - if idx := strings.LastIndex(line, "("); idx > 0 { - return line[:idx], nil +// Parses a single function from the given line. +// The line is in one of these formats: +// +// example.com/path/to/package.funcName(args...) +// example.com/path/to/package.(*typeName).funcName(args...) +// created by example.com/path/to/package.funcName in goroutine [...] +func parseFuncName(line string) (string, error) { + var name string + if after, ok := strings.CutPrefix(line, "created by "); ok { + // The function name is the part after "created by " + // and before " in goroutine [...]". + idx := strings.Index(after, " in goroutine") + if idx >= 0 { + name = after[:idx] + } + } else if idx := strings.LastIndexByte(line, '('); idx >= 0 { + // The function name is the part before the last '('. + name = line[:idx] + } + + if name == "" { + return "", fmt.Errorf("no function found: %q", line) } - return "", fmt.Errorf("no function found: %q", line) + + return name, nil } // parseGoStackHeader parses a stack header that looks like: diff --git a/internal/stack/stacks_test.go b/internal/stack/stacks_test.go index c324334..9615e8e 100644 --- a/internal/stack/stacks_test.go +++ b/internal/stack/stacks_test.go @@ -68,32 +68,43 @@ func TestAll(t *testing.T) { sort.Sort(byGoroutineID(got)) assert.Contains(t, got[0].Full(), "testing.(*T).Run") + assert.Contains(t, got[0].allFunctions, "testing.(*T).Run") + assert.Contains(t, got[1].Full(), "TestAll") + assert.Contains(t, got[1].allFunctions, "go.uber.org/goleak/internal/stack.TestAll") + for i := 0; i < 5; i++ { assert.Contains(t, got[2+i].Full(), "stack.waitForDone") } } func TestCurrent(t *testing.T) { + const pkgPrefix = "go.uber.org/goleak/internal/stack" + got := Current() assert.NotZero(t, got.ID(), "Should get non-zero goroutine id") assert.Equal(t, "running", got.State()) assert.Equal(t, "go.uber.org/goleak/internal/stack.getStackBuffer", got.FirstFunction()) wantFrames := []string{ - "stack.getStackBuffer", - "stack.getStacks", - "stack.Current", - "stack.Current", - "stack.TestCurrent", + "getStackBuffer", + "getStacks", + "Current", + "Current", + "TestCurrent", } all := got.Full() for _, frame := range wantFrames { - assert.Contains(t, all, frame) + name := pkgPrefix + "." + frame + assert.Contains(t, all, name) + assert.True(t, got.HasFunction(name), "missing in stack: %v\n%s", name, all) } assert.Contains(t, got.String(), "in state") assert.Contains(t, got.String(), "on top of the stack") + assert.True(t, got.HasFunction("testing.(*T).Run"), + "missing in stack: %v\n%s", "testing.(*T).Run", all) + // Ensure that we are not returning the buffer without slicing it // from getStackBuffer. if len(got.Full()) > 1024 { @@ -101,6 +112,21 @@ func TestCurrent(t *testing.T) { } } +func TestCurrentCreatedBy(t *testing.T) { + var stack Stack + done := make(chan struct{}) + go func() { + defer close(done) + stack = Current() + }() + <-done + + // This test function will be at the bottom of the stack + // as the function that created the goroutine. + assert.True(t, + stack.HasFunction("go.uber.org/goleak/internal/stack.TestCurrentCreatedBy")) +} + func TestAllLargeStack(t *testing.T) { const ( stackDepth = 100 @@ -134,6 +160,38 @@ func TestAllLargeStack(t *testing.T) { close(done) } +func TestParseFuncName(t *testing.T) { + tests := []struct { + name string + give string + want string + }{ + { + name: "function", + give: "example.com/foo/bar.baz()", + want: "example.com/foo/bar.baz", + }, + { + name: "method", + give: "example.com/foo/bar.(*baz).qux()", + want: "example.com/foo/bar.(*baz).qux", + }, + { + name: "created by", + give: "created by example.com/foo/bar.baz in goroutine 123", + want: "example.com/foo/bar.baz", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseFuncName(tt.give) + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + func TestParseStackErrors(t *testing.T) { tests := []struct { name string From 9e226f23b260265b7894588013d23af5efa26957 Mon Sep 17 00:00:00 2001 From: Abhinav Gupta Date: Sat, 21 Oct 2023 13:47:28 -0700 Subject: [PATCH 2/8] Support Go 1.20 "created by" lines In Go 1.20, the "created by" lines do not include the "in goroutine" portion. --- internal/stack/stacks.go | 4 +++- internal/stack/stacks_test.go | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/internal/stack/stacks.go b/internal/stack/stacks.go index a7411ec..0e8eb74 100644 --- a/internal/stack/stacks.go +++ b/internal/stack/stacks.go @@ -215,6 +215,7 @@ func getStackBuffer(all bool) []byte { // // example.com/path/to/package.funcName(args...) // example.com/path/to/package.(*typeName).funcName(args...) +// created by example.com/path/to/package.funcName // created by example.com/path/to/package.funcName in goroutine [...] func parseFuncName(line string) (string, error) { var name string @@ -223,8 +224,9 @@ func parseFuncName(line string) (string, error) { // and before " in goroutine [...]". idx := strings.Index(after, " in goroutine") if idx >= 0 { - name = after[:idx] + after = after[:idx] } + name = after } else if idx := strings.LastIndexByte(line, '('); idx >= 0 { // The function name is the part before the last '('. name = line[:idx] diff --git a/internal/stack/stacks_test.go b/internal/stack/stacks_test.go index 9615e8e..3ccdc21 100644 --- a/internal/stack/stacks_test.go +++ b/internal/stack/stacks_test.go @@ -177,7 +177,12 @@ func TestParseFuncName(t *testing.T) { want: "example.com/foo/bar.(*baz).qux", }, { - name: "created by", + name: "created by", // Go 1.20 + give: "created by example.com/foo/bar.baz", + want: "example.com/foo/bar.baz", + }, + { + name: "created by/in goroutine", // Go 1.21 give: "created by example.com/foo/bar.baz in goroutine 123", want: "example.com/foo/bar.baz", }, From a649b8e35bd33afdc4805d48d1d9071324d8e82e Mon Sep 17 00:00:00 2001 From: Abhinav Gupta Date: Sat, 21 Oct 2023 13:52:40 -0700 Subject: [PATCH 3/8] fix: Don't remove file paths from full traces `Full()` was accidentally dropping the file names from the full traces. --- internal/stack/stacks.go | 10 ++++++---- internal/stack/stacks_test.go | 3 +++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/internal/stack/stacks.go b/internal/stack/stacks.go index 0e8eb74..437239b 100644 --- a/internal/stack/stacks.go +++ b/internal/stack/stacks.go @@ -141,10 +141,6 @@ func (p *stackParser) parseStack(line string) (Stack, error) { funcs := make(map[string]struct{}) for p.scan.Scan() { line := p.scan.Text() - if len(line) == 0 { - continue - } - if strings.HasPrefix(line, "goroutine ") { // If we see the goroutine header, // it's the end of this stack. @@ -156,6 +152,10 @@ func (p *stackParser) parseStack(line string) (Stack, error) { fullStack.WriteString(line) fullStack.WriteByte('\n') // scanner trims the newline + if len(line) == 0 { + continue + } + funcName, err := parseFuncName(line) if err != nil { return Stack{}, fmt.Errorf("parse function: %w", err) @@ -174,6 +174,8 @@ func (p *stackParser) parseStack(line string) (Stack, error) { if p.scan.Scan() { bs := p.scan.Bytes() if len(bs) > 0 && bs[0] == '\t' { + fullStack.Write(bs) + fullStack.WriteByte('\n') continue } diff --git a/internal/stack/stacks_test.go b/internal/stack/stacks_test.go index 3ccdc21..9558ec6 100644 --- a/internal/stack/stacks_test.go +++ b/internal/stack/stacks_test.go @@ -105,6 +105,9 @@ func TestCurrent(t *testing.T) { assert.True(t, got.HasFunction("testing.(*T).Run"), "missing in stack: %v\n%s", "testing.(*T).Run", all) + assert.Contains(t, all, "stack/stacks_test.go", + "file name missing in stack:\n%s", all) + // Ensure that we are not returning the buffer without slicing it // from getStackBuffer. if len(got.Full()) > 1024 { From 6173c18aa0d93ec99e4533a8a2572dfa6988028a Mon Sep 17 00:00:00 2001 From: Abhinav Gupta Date: Sun, 22 Oct 2023 10:51:43 -0700 Subject: [PATCH 4/8] doc: Clarify defensiveness in parsing function names --- internal/stack/stacks.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/internal/stack/stacks.go b/internal/stack/stacks.go index 437239b..1136c5f 100644 --- a/internal/stack/stacks.go +++ b/internal/stack/stacks.go @@ -165,13 +165,14 @@ func (p *stackParser) parseStack(line string) (Stack, error) { firstFunction = funcName } - // The function name is usually followed by a line in the form: + // The function name followed by a line in the form: // // example.com/path/to/package/file.go:123 +0x123 // - // We don't care about the position, so we'll skip it, - // but only if it matches the expected format. + // We don't care about the position so we can skip this line. if p.scan.Scan() { + // Be defensive: + // Skip the line only if it starts with a tab. bs := p.scan.Bytes() if len(bs) > 0 && bs[0] == '\t' { fullStack.Write(bs) @@ -179,7 +180,8 @@ func (p *stackParser) parseStack(line string) (Stack, error) { continue } - // Put it back if it doesn't match. + // Put it back and let the next iteration handle it + // if it doesn't start with a tab. p.scan.Unscan() } } From af394ca8d273e0d3ba117f4bb74715060b4cc2c0 Mon Sep 17 00:00:00 2001 From: Abhinav Gupta Date: Sun, 22 Oct 2023 10:55:12 -0700 Subject: [PATCH 5/8] doc: Explain empty line skip --- internal/stack/stacks.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/stack/stacks.go b/internal/stack/stacks.go index 1136c5f..4c8af9b 100644 --- a/internal/stack/stacks.go +++ b/internal/stack/stacks.go @@ -153,6 +153,9 @@ func (p *stackParser) parseStack(line string) (Stack, error) { fullStack.WriteByte('\n') // scanner trims the newline if len(line) == 0 { + // Empty line usually marks the end of the stack + // but we don't want to have to rely on that. + // Just skip it. continue } From 7bdc2743e1cd649bb6b7ff5172fa10c2beb6df34 Mon Sep 17 00:00:00 2001 From: Abhinav Gupta Date: Sun, 22 Oct 2023 11:12:03 -0700 Subject: [PATCH 6/8] stack: "created by" is not part of the stack The function tha created a goroutine should not be considered part of its stack. However, we can use that entry to mark the end of a stack trace. --- internal/stack/stacks.go | 52 +++++++++++++++++++++++++++-------- internal/stack/stacks_test.go | 38 ++++++++++++++----------- 2 files changed, 62 insertions(+), 28 deletions(-) diff --git a/internal/stack/stacks.go b/internal/stack/stacks.go index 4c8af9b..241a9b8 100644 --- a/internal/stack/stacks.go +++ b/internal/stack/stacks.go @@ -159,13 +159,21 @@ func (p *stackParser) parseStack(line string) (Stack, error) { continue } - funcName, err := parseFuncName(line) + funcName, creator, err := parseFuncName(line) if err != nil { return Stack{}, fmt.Errorf("parse function: %w", err) } - funcs[funcName] = struct{}{} - if firstFunction == "" { - firstFunction = funcName + if !creator { + // A function is part of a goroutine's stack + // only if it's not a "created by" function. + // + // The creator function is part of a different stack. + // We don't care about it right now. + funcs[funcName] = struct{}{} + if firstFunction == "" { + firstFunction = funcName + } + } // The function name followed by a line in the form: @@ -180,12 +188,30 @@ func (p *stackParser) parseStack(line string) (Stack, error) { if len(bs) > 0 && bs[0] == '\t' { fullStack.Write(bs) fullStack.WriteByte('\n') - continue + } else { + // Put it back and let the next iteration handle it + // if it doesn't start with a tab. + p.scan.Unscan() } + } - // Put it back and let the next iteration handle it - // if it doesn't start with a tab. - p.scan.Unscan() + if creator { + // The "created by" line is the last line of the stack. + // We can stop parsing now. + // + // Note that if tracebackancestors=N is set, + // there may be more a traceback of the creator function + // following the "created by" line, + // but it should not be considered part of this stack. + // e.g., + // + // created by testing.(*T).Run in goroutine 1 + // /usr/lib/go/src/testing/testing.go:1648 +0x3ad + // [originating from goroutine 1]: + // testing.(*T).Run(...) + // /usr/lib/go/src/testing/testing.go:1649 +0x3ad + // + break } } @@ -224,8 +250,9 @@ func getStackBuffer(all bool) []byte { // example.com/path/to/package.(*typeName).funcName(args...) // created by example.com/path/to/package.funcName // created by example.com/path/to/package.funcName in goroutine [...] -func parseFuncName(line string) (string, error) { - var name string +// +// Also reports whether the line was a "created by" line. +func parseFuncName(line string) (name string, creator bool, err error) { if after, ok := strings.CutPrefix(line, "created by "); ok { // The function name is the part after "created by " // and before " in goroutine [...]". @@ -234,16 +261,17 @@ func parseFuncName(line string) (string, error) { after = after[:idx] } name = after + creator = true } else if idx := strings.LastIndexByte(line, '('); idx >= 0 { // The function name is the part before the last '('. name = line[:idx] } if name == "" { - return "", fmt.Errorf("no function found: %q", line) + return "", false, fmt.Errorf("no function found: %q", line) } - return name, nil + return name, creator, nil } // parseGoStackHeader parses a stack header that looks like: diff --git a/internal/stack/stacks_test.go b/internal/stack/stacks_test.go index 9558ec6..4f72049 100644 --- a/internal/stack/stacks_test.go +++ b/internal/stack/stacks_test.go @@ -102,9 +102,6 @@ func TestCurrent(t *testing.T) { assert.Contains(t, got.String(), "in state") assert.Contains(t, got.String(), "on top of the stack") - assert.True(t, got.HasFunction("testing.(*T).Run"), - "missing in stack: %v\n%s", "testing.(*T).Run", all) - assert.Contains(t, all, "stack/stacks_test.go", "file name missing in stack:\n%s", all) @@ -124,10 +121,15 @@ func TestCurrentCreatedBy(t *testing.T) { }() <-done - // This test function will be at the bottom of the stack - // as the function that created the goroutine. + // The test function created the goroutine + // so it won't be part of the stack. + assert.False(t, stack.HasFunction("go.uber.org/goleak/internal/stack.TestCurrentCreatedBy"), + "TestCurrentCreatedBy should not be in stack:\n%s", stack.Full()) + + // However, the nested function should be. assert.True(t, - stack.HasFunction("go.uber.org/goleak/internal/stack.TestCurrentCreatedBy")) + stack.HasFunction("go.uber.org/goleak/internal/stack.TestCurrentCreatedBy.func1"), + "TestCurrentCreatedBy.func1 is not in stack:\n%s", stack.Full()) } func TestAllLargeStack(t *testing.T) { @@ -165,9 +167,10 @@ func TestAllLargeStack(t *testing.T) { func TestParseFuncName(t *testing.T) { tests := []struct { - name string - give string - want string + name string + give string + want string + creator bool }{ { name: "function", @@ -180,22 +183,25 @@ func TestParseFuncName(t *testing.T) { want: "example.com/foo/bar.(*baz).qux", }, { - name: "created by", // Go 1.20 - give: "created by example.com/foo/bar.baz", - want: "example.com/foo/bar.baz", + name: "created by", // Go 1.20 + give: "created by example.com/foo/bar.baz", + want: "example.com/foo/bar.baz", + creator: true, }, { - name: "created by/in goroutine", // Go 1.21 - give: "created by example.com/foo/bar.baz in goroutine 123", - want: "example.com/foo/bar.baz", + name: "created by/in goroutine", // Go 1.21 + give: "created by example.com/foo/bar.baz in goroutine 123", + want: "example.com/foo/bar.baz", + creator: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := parseFuncName(tt.give) + got, creator, err := parseFuncName(tt.give) require.NoError(t, err) assert.Equal(t, tt.want, got) + assert.Equal(t, tt.creator, creator) }) } } From b4e742140eda43a1faa832b2ff74e4c509754e14 Mon Sep 17 00:00:00 2001 From: Abhinav Gupta Date: Sun, 22 Oct 2023 16:20:16 -0700 Subject: [PATCH 7/8] test: Add real stack traces To verify the stacktrace parsing logic, generate real stack traces under the following conditions: - Go 1.21 - Go 1.20 installed with gimme - Go 1.21 with tracebackancestors=10 set The test verifies that the parsed stack traces do not include functions that we did not expect to see in a goroutine's trace. --- internal/stack/stacks_test.go | 222 ++++++++++++++++++ internal/stack/testdata/Makefile | 27 +++ internal/stack/testdata/http.go | 48 ++++ internal/stack/testdata/http.go1.20.txt | 64 +++++ .../testdata/http.tracebackancestors.txt | 145 ++++++++++++ internal/stack/testdata/http.txt | 64 +++++ 6 files changed, 570 insertions(+) create mode 100644 internal/stack/testdata/Makefile create mode 100644 internal/stack/testdata/http.go create mode 100644 internal/stack/testdata/http.go1.20.txt create mode 100644 internal/stack/testdata/http.tracebackancestors.txt create mode 100644 internal/stack/testdata/http.txt diff --git a/internal/stack/stacks_test.go b/internal/stack/stacks_test.go index 4f72049..9c65115 100644 --- a/internal/stack/stacks_test.go +++ b/internal/stack/stacks_test.go @@ -21,6 +21,8 @@ package stack import ( + "os" + "path/filepath" "runtime" "sort" "strings" @@ -242,6 +244,226 @@ func TestParseStackErrors(t *testing.T) { } } +func TestParseStackFixtures(t *testing.T) { + type goroutine struct { + // ID must match the goroutine ID in the fixture. + // We use this to ensure that we are matching the right goroutine. + ID int + + State string + FirstFunction string + + HasFunctions []string // non-exhaustive, in any order + NotHasFunctions []string + } + + tests := []struct { + name string // file name inside testdata + stacks []goroutine // in any order + }{ + { + name: "http.txt", + stacks: []goroutine{ + { + ID: 1, + State: "running", + FirstFunction: "main.getStackBuffer", + HasFunctions: []string{ + "main.getStackBuffer", + "main.main", + }, + }, + { + ID: 4, + State: "IO wait", + FirstFunction: "internal/poll.runtime_pollWait", + HasFunctions: []string{ + "internal/poll.runtime_pollWait", + "net/http.Serve", + }, + NotHasFunctions: []string{"main.start"}, + }, + { + ID: 20, + State: "select", + FirstFunction: "net/http.(*persistConn).readLoop", + }, + { + ID: 21, + State: "select", + FirstFunction: "net/http.(*persistConn).writeLoop", + }, + { + ID: 8, + State: "IO wait", + FirstFunction: "internal/poll.runtime_pollWait", + HasFunctions: []string{ + "internal/poll.runtime_pollWait", + "net/http.(*conn).serve", + }, + NotHasFunctions: []string{"net/http.(*Server).Serve"}, + }, + }, + }, + { + name: "http.go1.20.txt", + stacks: []goroutine{ + { + ID: 1, + State: "running", + FirstFunction: "main.getStackBuffer", + HasFunctions: []string{ + "main.getStackBuffer", + "main.main", + }, + }, + { + ID: 20, + State: "IO wait", + FirstFunction: "internal/poll.runtime_pollWait", + HasFunctions: []string{ + "internal/poll.runtime_pollWait", + "net/http.(*Server).Serve", + }, + NotHasFunctions: []string{"main.start"}, + }, + { + ID: 24, + State: "select", + FirstFunction: "net/http.(*persistConn).readLoop", + }, + { + ID: 25, + State: "select", + FirstFunction: "net/http.(*persistConn).writeLoop", + }, + { + ID: 4, + State: "IO wait", + FirstFunction: "internal/poll.runtime_pollWait", + HasFunctions: []string{ + "internal/poll.runtime_pollWait", + "net/http.(*conn).serve", + }, + NotHasFunctions: []string{"net/http.(*Server).Serve"}, + }, + }, + }, + { + name: "http.tracebackancestors.txt", + stacks: []goroutine{ + { + ID: 1, + State: "running", + FirstFunction: "main.getStackBuffer", + HasFunctions: []string{ + "main.getStackBuffer", + "main.main", + }, + }, + { + ID: 20, + State: "IO wait", + FirstFunction: "internal/poll.runtime_pollWait", + HasFunctions: []string{ + "internal/poll.runtime_pollWait", + "net/http.Serve", + }, + NotHasFunctions: []string{ + "main.start", // created by + "main.main", // tracebackancestors + }, + }, + { + ID: 24, + State: "select", + FirstFunction: "net/http.(*persistConn).readLoop", + NotHasFunctions: []string{ + "net/http.(*Transport).dialConn", // created by + // tracebackancestors: + "net/http.(*Transport).dialConnFor", + "net/http.(*Transport).queueForDial", + "net/http.(*Client).Get", + "main.start", + "main.main", + }, + }, + { + ID: 4, + State: "IO wait", + FirstFunction: "internal/poll.runtime_pollWait", + HasFunctions: []string{ + "internal/poll.runtime_pollWait", + "net/http.(*conn).serve", + }, + NotHasFunctions: []string{ + "net/http.(*Server).Serve", // created by + // tracebackancestors: + "net/http.Serve", + "main.start", + "main.main", + }, + }, + { + ID: 25, + State: "select", + FirstFunction: "net/http.(*persistConn).writeLoop", + NotHasFunctions: []string{ + "net/http.(*Transport).dialConn", // created by + // tracebackancestors: + "net/http.(*Transport).dialConnFor", + "net/http.(*Transport).queueForDial", + "net/http.(*Client).Get", + "main.start", + "main.main", + }, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fixture, err := os.Open(filepath.Join("testdata", tt.name)) + require.NoError(t, err) + defer func() { + assert.NoError(t, fixture.Close()) + }() + + stacks, err := newStackParser(fixture).Parse() + require.NoError(t, err) + + stacksByID := make(map[int]Stack, len(stacks)) + for _, s := range stacks { + stacksByID[s.ID()] = s + } + + for _, wantStack := range tt.stacks { + gotStack, ok := stacksByID[wantStack.ID] + if !assert.True(t, ok, "missing stack %v", wantStack.ID) { + continue + } + delete(stacksByID, wantStack.ID) + + assert.Equal(t, wantStack.State, gotStack.State()) + assert.Equal(t, wantStack.FirstFunction, gotStack.FirstFunction()) + + for _, fn := range wantStack.HasFunctions { + assert.True(t, gotStack.HasFunction(fn), "missing in stack: %v\n%s", fn, gotStack.Full()) + } + + for _, fn := range wantStack.NotHasFunctions { + assert.False(t, gotStack.HasFunction(fn), "unexpected in stack: %v\n%s", fn, gotStack.Full()) + } + } + + for _, s := range stacksByID { + t.Errorf("unexpected stack:\n%s", s.Full()) + } + }) + } +} + func joinLines(lines ...string) string { return strings.Join(lines, "\n") + "\n" } diff --git a/internal/stack/testdata/Makefile b/internal/stack/testdata/Makefile new file mode 100644 index 0000000..76ffc15 --- /dev/null +++ b/internal/stack/testdata/Makefile @@ -0,0 +1,27 @@ +.DEFAULT_GOAL := all + +GO_VERSION = $(shell go version | cut -d' ' -f3) + +# Append to this list to add new stacks. +STACKS = + +# In Go 1.21, the output format was changed slightly. +# +# Generate a 1.20 version of the output +# only if we're running on Go 1.20. +ifneq (,$(findstring go1.20,$(GO_VERSION))) +STACKS += http.go1.20.txt +http.go1.20.txt: http.go + go run $< > $@ +else +STACKS += http.txt +http.txt: http.go + go run $< > $@ +endif + +STACKS += http.tracebackancestors.txt +http.tracebackancestors.txt: http.go + GODEBUG=tracebackancestors=10 go run $< > $@ + +.PHONY: all +all: $(STACKS) diff --git a/internal/stack/testdata/http.go b/internal/stack/testdata/http.go new file mode 100644 index 0000000..f258ade --- /dev/null +++ b/internal/stack/testdata/http.go @@ -0,0 +1,48 @@ +//go:build ignore + +package main + +import ( + "fmt" + "net" + "net/http" + "runtime" + "time" +) + +func main() { + if err := start(); err != nil { + panic(err) + } + + fmt.Println(string(getStackBuffer())) +} + +func start() error { + ln, err := net.Listen("tcp", ":0") + if err != nil { + return err + } + + go http.Serve(ln, nil) + + // Wait until HTTP server is ready. + url := "http://" + ln.Addr().String() + for i := 0; i < 10; i++ { + if _, err := http.Get(url); err == nil { + return nil + } + time.Sleep(100 * time.Millisecond) + } + + return fmt.Errorf("failed to start HTTP server") +} + +func getStackBuffer() []byte { + for i := 4096; ; i *= 2 { + buf := make([]byte, i) + if n := runtime.Stack(buf, true /* all */); n < i { + return buf[:n] + } + } +} diff --git a/internal/stack/testdata/http.go1.20.txt b/internal/stack/testdata/http.go1.20.txt new file mode 100644 index 0000000..fa7b9ce --- /dev/null +++ b/internal/stack/testdata/http.go1.20.txt @@ -0,0 +1,64 @@ +goroutine 1 [running]: +main.getStackBuffer() + /home/abg/src/goleak/internal/stack/testdata/http.go:44 +0x4f +main.main() + /home/abg/src/goleak/internal/stack/testdata/http.go:18 +0x2a + +goroutine 20 [IO wait]: +internal/poll.runtime_pollWait(0x7866b1e34f08, 0x72) + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/runtime/netpoll.go:306 +0x89 +internal/poll.(*pollDesc).wait(0xc0000dc000?, 0x16?, 0x0) + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32 +internal/poll.(*pollDesc).waitRead(...) + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0000dc000) + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/internal/poll/fd_unix.go:614 +0x2bd +net.(*netFD).accept(0xc0000dc000) + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/net/fd_unix.go:172 +0x35 +net.(*TCPListener).accept(0xc0000a00f0) + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/net/tcpsock_posix.go:148 +0x25 +net.(*TCPListener).Accept(0xc0000a00f0) + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/net/tcpsock.go:297 +0x3d +net/http.(*Server).Serve(0xc000076000, {0x73dbe0, 0xc0000a00f0}) + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/net/http/server.go:3059 +0x385 +net/http.Serve({0x73dbe0, 0xc0000a00f0}, {0x0?, 0x0}) + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/net/http/server.go:2581 +0x74 +created by main.start + /home/abg/src/goleak/internal/stack/testdata/http.go:27 +0x8e + +goroutine 24 [select]: +net/http.(*persistConn).readLoop(0xc0000b4480) + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/net/http/transport.go:2227 +0xd85 +created by net/http.(*Transport).dialConn + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/net/http/transport.go:1765 +0x16ea + +goroutine 25 [select]: +net/http.(*persistConn).writeLoop(0xc0000b4480) + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/net/http/transport.go:2410 +0xf2 +created by net/http.(*Transport).dialConn + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/net/http/transport.go:1766 +0x173d + +goroutine 4 [IO wait]: +internal/poll.runtime_pollWait(0x7866b1e34d28, 0x72) + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/runtime/netpoll.go:306 +0x89 +internal/poll.(*pollDesc).wait(0xc00007e000?, 0xc000106000?, 0x0) + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/internal/poll/fd_poll_runtime.go:84 +0x32 +internal/poll.(*pollDesc).waitRead(...) + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00007e000, {0xc000106000, 0x1000, 0x1000}) + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/internal/poll/fd_unix.go:167 +0x299 +net.(*netFD).Read(0xc00007e000, {0xc000106000?, 0x4a92e6?, 0x0?}) + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/net/fd_posix.go:55 +0x29 +net.(*conn).Read(0xc000014028, {0xc000106000?, 0x0?, 0xc0000781e8?}) + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/net/net.go:183 +0x45 +net/http.(*connReader).Read(0xc0000781e0, {0xc000106000, 0x1000, 0x1000}) + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/net/http/server.go:782 +0x171 +bufio.(*Reader).fill(0xc000104000) + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/bufio/bufio.go:106 +0xff +bufio.(*Reader).Peek(0xc000104000, 0x4) + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/bufio/bufio.go:144 +0x5d +net/http.(*conn).serve(0xc000100000, {0x73df98, 0xc0000780f0}) + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/net/http/server.go:2030 +0x77c +created by net/http.(*Server).Serve + /home/abg/.gimme/versions/go1.20.10.linux.amd64/src/net/http/server.go:3089 +0x5ed + diff --git a/internal/stack/testdata/http.tracebackancestors.txt b/internal/stack/testdata/http.tracebackancestors.txt new file mode 100644 index 0000000..2e65512 --- /dev/null +++ b/internal/stack/testdata/http.tracebackancestors.txt @@ -0,0 +1,145 @@ +goroutine 1 [running]: +main.getStackBuffer() + /home/abg/src/goleak/internal/stack/testdata/http.go:44 +0x49 +main.main() + /home/abg/src/goleak/internal/stack/testdata/http.go:18 +0x1d + +goroutine 20 [IO wait]: +internal/poll.runtime_pollWait(0x7c3a3d619e48, 0x72) + /usr/lib/go/src/runtime/netpoll.go:343 +0x85 +internal/poll.(*pollDesc).wait(0xc0000da000?, 0x16?, 0x0) + /usr/lib/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 +internal/poll.(*pollDesc).waitRead(...) + /usr/lib/go/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc0000da000) + /usr/lib/go/src/internal/poll/fd_unix.go:611 +0x2ac +net.(*netFD).accept(0xc0000da000) + /usr/lib/go/src/net/fd_unix.go:172 +0x29 +net.(*TCPListener).accept(0xc0000ba0c0) + /usr/lib/go/src/net/tcpsock_posix.go:152 +0x1e +net.(*TCPListener).Accept(0xc0000ba0c0) + /usr/lib/go/src/net/tcpsock.go:315 +0x30 +net/http.(*Server).Serve(0xc000078000, {0x738a20, 0xc0000ba0c0}) + /usr/lib/go/src/net/http/server.go:3056 +0x364 +net/http.Serve({0x738a20, 0xc0000ba0c0}, {0x0?, 0x0}) + /usr/lib/go/src/net/http/server.go:2595 +0x6c +created by main.start in goroutine 1 + /home/abg/src/goleak/internal/stack/testdata/http.go:27 +0x87 +[originating from goroutine 1]: +main.start(...) + /home/abg/src/goleak/internal/stack/testdata/http.go:30 +0x87 +main.main(...) + /home/abg/src/goleak/internal/stack/testdata/http.go:14 +0x13 + +goroutine 24 [select]: +net/http.(*persistConn).readLoop(0xc0000be480) + /usr/lib/go/src/net/http/transport.go:2238 +0xd25 +created by net/http.(*Transport).dialConn in goroutine 21 + /usr/lib/go/src/net/http/transport.go:1776 +0x169f +[originating from goroutine 21]: +net/http.(*Transport).dialConn(...) + /usr/lib/go/src/net/http/transport.go:1777 +0x169f +net/http.(*Transport).dialConnFor(...) + /usr/lib/go/src/net/http/transport.go:1469 +0x9f +created by net/http.(*Transport).queueForDial + /usr/lib/go/src/net/http/transport.go:1436 +0x3cb +[originating from goroutine 1]: +net/http.(*Transport).queueForDial(...) + /usr/lib/go/src/net/http/transport.go:1437 +0x3cb +net/http.(*Request).Context(...) + /usr/lib/go/src/net/http/request.go:346 +0x4c9 +net/http.(*Transport).roundTrip(...) + /usr/lib/go/src/net/http/transport.go:591 +0x73a +net/http.(*Transport).RoundTrip(...) + /usr/lib/go/src/net/http/roundtrip.go:17 +0x13 +net/http.send(...) + /usr/lib/go/src/net/http/client.go:260 +0x606 +net/http.(*Client).send(...) + /usr/lib/go/src/net/http/client.go:182 +0x98 +net/http.(*Client).do(...) + /usr/lib/go/src/net/http/client.go:724 +0x912 +net/http.(*Client).Get(...) + /usr/lib/go/src/net/http/client.go:488 +0x5f +net/http.(*Client).Get(...) + /usr/lib/go/src/net/http/client.go:488 +0x60 +main.start(...) + /home/abg/src/goleak/internal/stack/testdata/http.go:32 +0x111 +main.start(...) + /home/abg/src/goleak/internal/stack/testdata/http.go:32 +0x112 +main.main(...) + /home/abg/src/goleak/internal/stack/testdata/http.go:14 +0x13 + +goroutine 4 [IO wait]: +internal/poll.runtime_pollWait(0x7c3a3d619d50, 0x72) + /usr/lib/go/src/runtime/netpoll.go:343 +0x85 +internal/poll.(*pollDesc).wait(0xc00007e000?, 0xc000106000?, 0x0) + /usr/lib/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 +internal/poll.(*pollDesc).waitRead(...) + /usr/lib/go/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc00007e000, {0xc000106000, 0x1000, 0x1000}) + /usr/lib/go/src/internal/poll/fd_unix.go:164 +0x27a +net.(*netFD).Read(0xc00007e000, {0xc000106000?, 0x4a8965?, 0x0?}) + /usr/lib/go/src/net/fd_posix.go:55 +0x25 +net.(*conn).Read(0xc000046018, {0xc000106000?, 0x0?, 0xc000064248?}) + /usr/lib/go/src/net/net.go:179 +0x45 +net/http.(*connReader).Read(0xc000064240, {0xc000106000, 0x1000, 0x1000}) + /usr/lib/go/src/net/http/server.go:791 +0x14b +bufio.(*Reader).fill(0xc000104000) + /usr/lib/go/src/bufio/bufio.go:113 +0x103 +bufio.(*Reader).Peek(0xc000104000, 0x4) + /usr/lib/go/src/bufio/bufio.go:151 +0x53 +net/http.(*conn).serve(0xc000100000, {0x739108, 0xc000064150}) + /usr/lib/go/src/net/http/server.go:2044 +0x75c +created by net/http.(*Server).Serve in goroutine 20 + /usr/lib/go/src/net/http/server.go:3086 +0x5cb +[originating from goroutine 20]: +net/http.(*Server).Serve(...) + /usr/lib/go/src/net/http/server.go:3086 +0x5cb +net/http.Serve(...) + /usr/lib/go/src/net/http/server.go:2595 +0x6c +created by main.start + /home/abg/src/goleak/internal/stack/testdata/http.go:27 +0x87 +[originating from goroutine 1]: +main.start(...) + /home/abg/src/goleak/internal/stack/testdata/http.go:30 +0x87 +main.main(...) + /home/abg/src/goleak/internal/stack/testdata/http.go:14 +0x13 + +goroutine 25 [select]: +net/http.(*persistConn).writeLoop(0xc0000be480) + /usr/lib/go/src/net/http/transport.go:2421 +0xe5 +created by net/http.(*Transport).dialConn in goroutine 21 + /usr/lib/go/src/net/http/transport.go:1777 +0x16f1 +[originating from goroutine 21]: +net/http.(*Transport).dialConn(...) + /usr/lib/go/src/net/http/transport.go:1778 +0x16f1 +net/http.(*Transport).dialConnFor(...) + /usr/lib/go/src/net/http/transport.go:1469 +0x9f +created by net/http.(*Transport).queueForDial + /usr/lib/go/src/net/http/transport.go:1436 +0x3cb +[originating from goroutine 1]: +net/http.(*Transport).queueForDial(...) + /usr/lib/go/src/net/http/transport.go:1437 +0x3cb +net/http.(*Request).Context(...) + /usr/lib/go/src/net/http/request.go:346 +0x4c9 +net/http.(*Transport).roundTrip(...) + /usr/lib/go/src/net/http/transport.go:591 +0x73a +net/http.(*Transport).RoundTrip(...) + /usr/lib/go/src/net/http/roundtrip.go:17 +0x13 +net/http.send(...) + /usr/lib/go/src/net/http/client.go:260 +0x606 +net/http.(*Client).send(...) + /usr/lib/go/src/net/http/client.go:182 +0x98 +net/http.(*Client).do(...) + /usr/lib/go/src/net/http/client.go:724 +0x912 +net/http.(*Client).Get(...) + /usr/lib/go/src/net/http/client.go:488 +0x5f +net/http.(*Client).Get(...) + /usr/lib/go/src/net/http/client.go:488 +0x60 +main.start(...) + /home/abg/src/goleak/internal/stack/testdata/http.go:32 +0x111 +main.start(...) + /home/abg/src/goleak/internal/stack/testdata/http.go:32 +0x112 +main.main(...) + /home/abg/src/goleak/internal/stack/testdata/http.go:14 +0x13 + diff --git a/internal/stack/testdata/http.txt b/internal/stack/testdata/http.txt new file mode 100644 index 0000000..c92cbd8 --- /dev/null +++ b/internal/stack/testdata/http.txt @@ -0,0 +1,64 @@ +goroutine 1 [running]: +main.getStackBuffer() + /home/abg/src/goleak/internal/stack/testdata/http.go:44 +0x49 +main.main() + /home/abg/src/goleak/internal/stack/testdata/http.go:18 +0x1d + +goroutine 4 [IO wait]: +internal/poll.runtime_pollWait(0x7bf130ae7ea0, 0x72) + /usr/lib/go/src/runtime/netpoll.go:343 +0x85 +internal/poll.(*pollDesc).wait(0xc000132000?, 0x4?, 0x0) + /usr/lib/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 +internal/poll.(*pollDesc).waitRead(...) + /usr/lib/go/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Accept(0xc000132000) + /usr/lib/go/src/internal/poll/fd_unix.go:611 +0x2ac +net.(*netFD).accept(0xc000132000) + /usr/lib/go/src/net/fd_unix.go:172 +0x29 +net.(*TCPListener).accept(0xc0000600e0) + /usr/lib/go/src/net/tcpsock_posix.go:152 +0x1e +net.(*TCPListener).Accept(0xc0000600e0) + /usr/lib/go/src/net/tcpsock.go:315 +0x30 +net/http.(*Server).Serve(0xc00008c000, {0x738a20, 0xc0000600e0}) + /usr/lib/go/src/net/http/server.go:3056 +0x364 +net/http.Serve({0x738a20, 0xc0000600e0}, {0x0?, 0x0}) + /usr/lib/go/src/net/http/server.go:2595 +0x6c +created by main.start in goroutine 1 + /home/abg/src/goleak/internal/stack/testdata/http.go:27 +0x87 + +goroutine 20 [select]: +net/http.(*persistConn).readLoop(0xc000112480) + /usr/lib/go/src/net/http/transport.go:2238 +0xd25 +created by net/http.(*Transport).dialConn in goroutine 5 + /usr/lib/go/src/net/http/transport.go:1776 +0x169f + +goroutine 21 [select]: +net/http.(*persistConn).writeLoop(0xc000112480) + /usr/lib/go/src/net/http/transport.go:2421 +0xe5 +created by net/http.(*Transport).dialConn in goroutine 5 + /usr/lib/go/src/net/http/transport.go:1777 +0x16f1 + +goroutine 8 [IO wait]: +internal/poll.runtime_pollWait(0x7bf130ae7cb0, 0x72) + /usr/lib/go/src/runtime/netpoll.go:343 +0x85 +internal/poll.(*pollDesc).wait(0xc000132200?, 0xc000142000?, 0x0) + /usr/lib/go/src/internal/poll/fd_poll_runtime.go:84 +0x27 +internal/poll.(*pollDesc).waitRead(...) + /usr/lib/go/src/internal/poll/fd_poll_runtime.go:89 +internal/poll.(*FD).Read(0xc000132200, {0xc000142000, 0x1000, 0x1000}) + /usr/lib/go/src/internal/poll/fd_unix.go:164 +0x27a +net.(*netFD).Read(0xc000132200, {0xc000142000?, 0x4a8965?, 0x0?}) + /usr/lib/go/src/net/fd_posix.go:55 +0x25 +net.(*conn).Read(0xc000044070, {0xc000142000?, 0x0?, 0xc00007ad28?}) + /usr/lib/go/src/net/net.go:179 +0x45 +net/http.(*connReader).Read(0xc00007ad20, {0xc000142000, 0x1000, 0x1000}) + /usr/lib/go/src/net/http/server.go:791 +0x14b +bufio.(*Reader).fill(0xc000102540) + /usr/lib/go/src/bufio/bufio.go:113 +0x103 +bufio.(*Reader).Peek(0xc000102540, 0x4) + /usr/lib/go/src/bufio/bufio.go:151 +0x53 +net/http.(*conn).serve(0xc000134240, {0x739108, 0xc00008e0f0}) + /usr/lib/go/src/net/http/server.go:2044 +0x75c +created by net/http.(*Server).Serve in goroutine 4 + /usr/lib/go/src/net/http/server.go:3086 +0x5cb + From 5b4de071abf88b8394ec7a9b6674ea3c28aefc07 Mon Sep 17 00:00:00 2001 From: Abhinav Gupta Date: Sun, 22 Oct 2023 21:13:57 -0700 Subject: [PATCH 8/8] parseStack: unit test for corner cases --- internal/stack/stacks_test.go | 75 +++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/internal/stack/stacks_test.go b/internal/stack/stacks_test.go index 9c65115..156c2aa 100644 --- a/internal/stack/stacks_test.go +++ b/internal/stack/stacks_test.go @@ -208,6 +208,81 @@ func TestParseFuncName(t *testing.T) { } } +func TestParseStack(t *testing.T) { + tests := []struct { + name string + give string + + id int + state string + firstFunc string + funcs []string + }{ + { + name: "running", + give: joinLines( + "goroutine 1 [running]:", + "example.com/foo/bar.baz()", + " example.com/foo/bar.go:123", + ), + id: 1, + state: "running", + firstFunc: "example.com/foo/bar.baz", + funcs: []string{"example.com/foo/bar.baz"}, + }, + { + name: "without position", + give: joinLines( + "goroutine 1 [running]:", + "example.com/foo/bar.baz()", + // Oops, no "file:line" entry for this function. + "example.com/foo/bar.qux()", + " example.com/foo/bar.go:456", + ), + id: 1, + state: "running", + firstFunc: "example.com/foo/bar.baz", + funcs: []string{ + "example.com/foo/bar.baz", + "example.com/foo/bar.qux", + }, + }, + { + name: "created by", + give: joinLines( + "goroutine 1 [running]:", + "example.com/foo/bar.baz()", + " example.com/foo/bar.go:123", + "created by example.com/foo/bar.qux", + " example.com/foo/bar.go:456", + ), + id: 1, + state: "running", + firstFunc: "example.com/foo/bar.baz", + funcs: []string{ + "example.com/foo/bar.baz", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stacks, err := newStackParser(strings.NewReader(tt.give)).Parse() + require.NoError(t, err) + require.Len(t, stacks, 1) + + stack := stacks[0] + assert.Equal(t, tt.id, stack.ID()) + assert.Equal(t, tt.state, stack.State()) + assert.Equal(t, tt.firstFunc, stack.FirstFunction()) + for _, fn := range tt.funcs { + assert.True(t, stack.HasFunction(fn), + "missing in stack: %v\n%s", fn, stack.Full()) + } + }) + } +} + func TestParseStackErrors(t *testing.T) { tests := []struct { name string