Enable guessing GOROOT and GOPATH

Add -rebase=false as an option to disable this functionality.

Fixes #35.

internal/main.go

@@ -69,7 +69,10 @@ func writeToConsole(out io.Writer, p *stack.Palette, buckets stack.Buckets, full
 // process copies stdin to stdout and processes any "panic: " line found.
 //
 // If html is used, a stack trace is written to this file instead.
-func process(in io.Reader, out io.Writer, p *stack.Palette, s stack.Similarity, fullPath, parse bool, html string) error {
+func process(in io.Reader, out io.Writer, p *stack.Palette, s stack.Similarity, fullPath, parse, rebase bool, html string) error {
+	if !rebase {
+		stack.NoRebase()
+	}
 	goroutines, err := stack.ParseDump(in, out)
 	if err != nil {
 		return err
@@ -99,6 +102,7 @@ func showBanner() bool {
 func Main() error {
 	aggressive := flag.Bool("aggressive", false, "Aggressive deduplication including non pointers")
 	parse := flag.Bool("parse", true, "Parses source files to deduct types; use -parse=false to work around bugs in source parser")
+	rebase := flag.Bool("rebase", true, "Guess GOROOT and GOPATH")
 	verboseFlag := flag.Bool("v", false, "Enables verbose logging output")
 	// Console only.
 	fullPath := flag.Bool("full-path", false, "Print full sources path")
@@ -155,5 +159,5 @@ func Main() error {
 		return errors.New("pipe from stdin or specify a single file")
 	}
 
-	return process(in, out, p, s, *fullPath, *parse, *html)
+	return process(in, out, p, s, *fullPath, *parse, *rebase, *html)
 }

internal/main_test.go

@@ -48,7 +48,7 @@ var data = []string{
 
 func TestProcess(t *testing.T) {
 	out := &bytes.Buffer{}
-	err := process(bytes.NewBufferString(strings.Join(data, "\n")), out, &defaultPalette, stack.AnyPointer, false, false, "")
+	err := process(bytes.NewBufferString(strings.Join(data, "\n")), out, &defaultPalette, stack.AnyPointer, false, false, true, "")
 	ut.AssertEqual(t, nil, err)
 	expected := []string{
 		"panic: runtime error: index out of range",
@@ -73,7 +73,7 @@ func TestProcess(t *testing.T) {
 
 func TestProcessFullPath(t *testing.T) {
 	out := &bytes.Buffer{}
-	err := process(bytes.NewBufferString(strings.Join(data, "\n")), out, &defaultPalette, stack.AnyValue, true, false, "")
+	err := process(bytes.NewBufferString(strings.Join(data, "\n")), out, &defaultPalette, stack.AnyValue, true, false, true, "")
 	ut.AssertEqual(t, nil, err)
 	expected := []string{
 		"panic: runtime error: index out of range",
@@ -98,7 +98,7 @@ func TestProcessFullPath(t *testing.T) {
 
 func TestProcessNoColor(t *testing.T) {
 	out := &bytes.Buffer{}
-	err := process(bytes.NewBufferString(strings.Join(data, "\n")), out, &stack.Palette{}, stack.AnyPointer, false, false, "")
+	err := process(bytes.NewBufferString(strings.Join(data, "\n")), out, &stack.Palette{}, stack.AnyPointer, false, false, true, "")
 	ut.AssertEqual(t, nil, err)
 	expected := []string{
 		"panic: runtime error: index out of range",

stack/source.go

@@ -49,7 +49,7 @@ func (c *cache) augmentGoroutine(goroutine *Goroutine) {
 	// For each call site, look at the next call and populate it. Then we can
 	// walk back and reformat things.
 	for i := range goroutine.Stack.Calls {
-		c.load(goroutine.Stack.Calls[i].SourcePath)
+		c.load(goroutine.Stack.Calls[i].LocalSourcePath())
 	}
 
 	// Once all loaded, we can look at the next call when available.
@@ -101,7 +101,7 @@ func (c *cache) load(fileName string) {
 }
 
 func (c *cache) getFuncAST(call *Call) *ast.FuncDecl {
-	if p := c.parsed[call.SourcePath]; p != nil {
+	if p := c.parsed[call.LocalSourcePath()]; p != nil {
 		return p.getFuncAST(call.Func.Name(), call.Line)
 	}
 	return nil

stack/source_test.go

@@ -18,6 +18,7 @@ import (
 )
 
 func TestAugment(t *testing.T) {
+	defer reset()
 	data := []struct {
 		name     string
 		input    string
@@ -430,20 +431,24 @@ func TestAugment(t *testing.T) {
 
 	for i, line := range data {
 		extra := bytes.Buffer{}
-		_, content := getCrash(t, line.input)
+		_, content, clean := getCrash(t, line.input)
+		reset()
 		goroutines, err := ParseDump(bytes.NewBuffer(content), &extra)
 		if err != nil {
+			clean()
 			t.Fatalf("failed to parse input for test %s: %v", line.name, err)
 		}
 		// On go1.4, there's one less space.
 		actual := extra.String()
 		if actual != "panic: ooh\n\nexit status 2\n" && actual != "panic: ooh\nexit status 2\n" {
+			clean()
 			t.Fatalf("Unexpected panic output:\n%#v", actual)
 		}
 		s := goroutines[0].Signature.Stack
 		t.Logf("Test: %v", line.name)
 		zapPointers(t, line.name, &line.expected, &s)
 		zapPaths(&s)
+		clean()
 		ut.AssertEqualIndex(t, i, line.expected, s)
 	}
 }
@@ -502,28 +507,32 @@ func overrideEnv(env []string, key, value string) []string {
 	return append(env, prefix+value)
 }
 
-func getCrash(t *testing.T, content string) (string, []byte) {
+func getCrash(t *testing.T, content string) (string, []byte, func()) {
+	//p := getGOPATHs()
+	//name, err := ioutil.TempDir(filepath.Join(p[0], "src"), "panicparse")
 	name, err := ioutil.TempDir("", "panicparse")
 	if err != nil {
 		t.Fatalf("failed to create temporary directory: %v", err)
 	}
-	defer func() {
+	clean := func() {
 		if err := os.RemoveAll(name); err != nil {
 			t.Fatalf("failed to remove temporary directory %q: %v", name, err)
 		}
-	}()
+	}
 	main := filepath.Join(name, "main.go")
 	if err := ioutil.WriteFile(main, []byte(content), 0500); err != nil {
+		clean()
 		t.Fatalf("failed to write %q: %v", main, err)
 	}
 	cmd := exec.Command("go", "run", main)
 	// Use the Go 1.4 compatible format.
 	cmd.Env = overrideEnv(os.Environ(), "GOTRACEBACK", "1")
 	out, err := cmd.CombinedOutput()
 	if err == nil {
+		clean()
 		t.Fatal("expected error since this is supposed to crash")
 	}
-	return main, out
+	return main, out, clean
 }
 
 // zapPointers zaps out pointers.

stack/stack.go

@@ -14,9 +14,11 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"log"
 	"math"
 	"net/url"
 	"os"
+	"os/user"
 	"path/filepath"
 	"regexp"
 	"runtime"
@@ -60,13 +62,24 @@ var (
 	reCreated = regexp.MustCompile("^created by (.+)\r?\n$")
 	reFunc    = regexp.MustCompile("^(.+)\\((.*)\\)\r?\n$")
 	reElided  = regexp.MustCompile("^\\.\\.\\.additional frames elided\\.\\.\\.\r?\n$")
-	// Include frequent GOROOT value on Windows, distro provided and user
-	// installed path. This simplifies the user's life when processing a trace
-	// generated on another VM.
-	// TODO(maruel): Guess the path automatically via traces containing the
-	// 'runtime' package, which is very frequent. This would be "less bad" than
-	// throwing up random values at the parser.
-	goroots = []string{runtime.GOROOT(), "c:/go", "/usr/lib/go", "/usr/local/go"}
+
+	// TODO(maruel): This is a global state, affected by ParseDump(). This will
+	// be refactored in v2.
+
+	// goroot is the GOROOT as detected in the traceback, not the on the host.
+	//
+	// It can be empty if no root was determined, for example the traceback
+	// contains only non-stdlib source references.
+	goroot string
+	// gopaths is the GOPATH as detected in the traceback, with the value being
+	// the corresponding path mapped to the host.
+	//
+	// It can be empty if only stdlib code is in the traceback or if no local
+	// sources were matched up. In the general case there is only one.
+	gopaths map[string]string
+	// Corresponding local values on the host.
+	localgoroot  = runtime.GOROOT()
+	localgopaths = getGOPATHs()
 )
 
 // Function is a function call.
@@ -235,7 +248,7 @@ func (a *Args) Merge(r *Args) Args {
 
 // Call is an item in the stack trace.
 type Call struct {
-	SourcePath string   // Full path name of the source file
+	SourcePath string   // Full path name of the source file as seen in the trace
 	Line       int      // Line number
 	Func       Function // Fully qualified function name (encoded).
 	Args       Args     // Call arguments
@@ -272,7 +285,23 @@ func (c *Call) SourceLine() string {
 	return fmt.Sprintf("%s:%d", c.SourceName(), c.Line)
 }
 
+// LocalSourcePath is the full path name of the source file as seen in the host.
+func (c *Call) LocalSourcePath() string {
+	// TODO(maruel): Call needs members goroot and gopaths.
+	if strings.HasPrefix(c.SourcePath, goroot) {
+		return filepath.Join(localgoroot, c.SourcePath[len(goroot):])
+	}
+	for prefix, dest := range gopaths {
+		if strings.HasPrefix(c.SourcePath, prefix) {
+			return filepath.Join(dest, c.SourcePath[len(prefix):])
+		}
+	}
+	return c.SourcePath
+}
+
 // FullSourceLine returns "/path/to/source.go:line".
+//
+// This file path is mutated to look like the local path.
 func (c *Call) FullSourceLine() string {
 	return fmt.Sprintf("%s:%d", c.SourcePath, c.Line)
 }
@@ -287,13 +316,8 @@ const testMainSource = "_test" + string(os.PathSeparator) + "_testmain.go"
 // IsStdlib returns true if it is a Go standard library function. This includes
 // the 'go test' generated main executable.
 func (c *Call) IsStdlib() bool {
-	for _, goroot := range goroots {
-		if strings.HasPrefix(c.SourcePath, goroot) {
-			return true
-		}
-	}
 	// Consider _test/_testmain.go as stdlib since it's injected by "go test".
-	return c.PkgSource() == testMainSource
+	return (goroot != "" && strings.HasPrefix(c.SourcePath, goroot)) || c.PkgSource() == testMainSource
 }
 
 // IsPkgMain returns true if it is in the main package.
@@ -663,9 +687,26 @@ func ParseDump(r io.Reader, out io.Writer) ([]Goroutine, error) {
 		goroutine = nil
 	}
 	nameArguments(goroutines)
+	// Mutate global state.
+	// TODO(maruel): Make this part of the context instead of a global.
+	if goroot == "" {
+		findRoots(goroutines)
+	}
 	return goroutines, scanner.Err()
 }
 
+// NoRebase disables GOROOT and GOPATH guessing in ParseDump().
+//
+// BUG: This function will be removed in v2, as ParseDump() will accept a flag
+// explicitly.
+func NoRebase() {
+	goroot = runtime.GOROOT()
+	gopaths = map[string]string{}
+	for _, p := range getGOPATHs() {
+		gopaths[p] = p
+	}
+}
+
 // Private stuff.
 
 func nameArguments(goroutines []Goroutine) {
@@ -725,6 +766,133 @@ func nameArguments(goroutines []Goroutine) {
 	}
 }
 
+// hasPathPrefix returns true if any of s is the prefix of p.
+func hasPathPrefix(p string, s map[string]string) bool {
+	for prefix := range s {
+		if strings.HasPrefix(p, prefix+"/") {
+			return true
+		}
+	}
+	return false
+}
+
+// getFiles returns all the source files deduped and ordered.
+func getFiles(goroutines []Goroutine) []string {
+	files := map[string]struct{}{}
+	for _, g := range goroutines {
+		for _, c := range g.Stack.Calls {
+			files[c.SourcePath] = struct{}{}
+		}
+	}
+	out := make([]string, 0, len(files))
+	for f := range files {
+		out = append(out, f)
+	}
+	sort.Strings(out)
+	return out
+}
+
+// splitPath splits a path into its components.
+//
+// The first item has its initial path separator kept.
+func splitPath(p string) []string {
+	if p == "" {
+		return nil
+	}
+	var out []string
+	s := ""
+	for _, c := range p {
+		if c != '/' || (len(out) == 0 && strings.Count(s, "/") == len(s)) {
+			s += string(c)
+		} else if s != "" {
+			out = append(out, s)
+			s = ""
+		}
+	}
+	if s != "" {
+		out = append(out, s)
+	}
+	return out
+}
+
+// isFile returns true if the path is a valid file.
+func isFile(p string) bool {
+	// TODO(maruel): Is it faster to open the file or to stat it? Worth a perf
+	// test on Windows.
+	i, err := os.Stat(p)
+	return err == nil && !i.IsDir()
+}
+
+// isRootIn returns a root if the file split in parts is rooted in root.
+func rootedIn(root string, parts []string) string {
+	//log.Printf("rootIn(%s, %v)", root, parts)
+	for i := 1; i < len(parts); i++ {
+		suffix := filepath.Join(parts[i:]...)
+		if isFile(filepath.Join(root, suffix)) {
+			return filepath.Join(parts[:i]...)
+		}
+	}
+	return ""
+}
+
+// findRoots sets global variables goroot and gopath.
+//
+// TODO(maruel): In v2, it will be a property of the new struct that will
+// contain the goroutines.
+func findRoots(goroutines []Goroutine) {
+	gopaths = map[string]string{}
+	for _, f := range getFiles(goroutines) {
+		// TODO(maruel): Could a stack dump have mixed cases? I think it's
+		// possible, need to confirm and handle.
+		//log.Printf("  Analyzing %s", f)
+		if goroot != "" && strings.HasPrefix(f, goroot+"/") {
+			continue
+		}
+		if gopaths != nil && hasPathPrefix(f, gopaths) {
+			continue
+		}
+		parts := splitPath(f)
+		if goroot == "" {
+			if r := rootedIn(localgoroot, parts); r != "" {
+				goroot = r
+				log.Printf("Found GOROOT=%s", goroot)
+				continue
+			}
+		}
+		found := false
+		for _, l := range localgopaths {
+			if r := rootedIn(l, parts); r != "" {
+				log.Printf("Found GOPATH=%s", r)
+				gopaths[r] = l
+				found = true
+				break
+			}
+		}
+		if !found {
+			// If the source is not found, just too bad.
+			//log.Printf("Failed to find locally: %s / %s", f, goroot)
+		}
+	}
+}
+
+func getGOPATHs() []string {
+	var out []string
+	for _, v := range filepath.SplitList(os.Getenv("GOPATH")) {
+		// Disallow non-absolute paths?
+		if v != "" {
+			out = append(out, v)
+		}
+	}
+	if len(out) == 0 {
+		u, err := user.Current()
+		if err != nil {
+			panic(err)
+		}
+		out = []string{u.HomeDir + "go"}
+	}
+	return out
+}
+
 type uint64Slice []uint64
 
 func (a uint64Slice) Len() int           { return len(a) }