Performance tweaks (#127)

.data/Dockerfile

@@ -1,6 +1,6 @@
 FROM alpine:latest
 ADD README.md /somefile.txt
-RUN mkdir /root/example
+RUN mkdir -p /root/example/really/nested
 RUN cp /somefile.txt /root/example/somefile1.txt
 RUN chmod 444 /root/example/somefile1.txt
 RUN cp /somefile.txt /root/example/somefile2.txt

cmd/analyze.go

@@ -2,9 +2,9 @@ package cmd
 
 import (
 	"fmt"
-
 	"github.com/fatih/color"
 	"github.com/spf13/cobra"
+	"github.com/wagoodman/dive/filetree"
 	"github.com/wagoodman/dive/image"
 	"github.com/wagoodman/dive/ui"
 	"github.com/wagoodman/dive/utils"
@@ -33,8 +33,13 @@ func doAnalyzeCmd(cmd *cobra.Command, args []string) {
 		utils.Exit(1)
 	}
 	color.New(color.Bold).Println("Analyzing Image")
+	result := fetchAndAnalyze(userImage)
+
+	fmt.Println("  Building cache...")
+	cache := filetree.NewFileTreeCache(result.RefTrees)
+	cache.Build()
 
-	ui.Run(fetchAndAnalyze(userImage))
+	ui.Run(result, cache)
 }
 
 func fetchAndAnalyze(imageID string) *image.AnalysisResult {

cmd/build.go

@@ -1,9 +1,11 @@
 package cmd
 
 import (
+	"fmt"
 	"github.com/fatih/color"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"github.com/wagoodman/dive/filetree"
 	"github.com/wagoodman/dive/ui"
 	"github.com/wagoodman/dive/utils"
 	"io/ioutil"
@@ -46,6 +48,11 @@ func doBuildCmd(cmd *cobra.Command, args []string) {
 	}
 
 	color.New(color.Bold).Println("Analyzing Image")
+	result := fetchAndAnalyze(string(imageId))
 
-	ui.Run(fetchAndAnalyze(string(imageId)))
+	fmt.Println("  Building cache...")
+	cache := filetree.NewFileTreeCache(result.RefTrees)
+	cache.Build()
+
+	ui.Run(result, cache)
 }

cmd/root.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/wagoodman/dive/filetree"
 	"github.com/wagoodman/dive/utils"
 	"io/ioutil"
 	"os"
@@ -110,6 +111,9 @@ func initConfig() {
 	if err := viper.ReadInConfig(); err == nil {
 		fmt.Println("Using config file:", viper.ConfigFileUsed())
 	}
+
+	// set global defaults (for performance)
+	filetree.GlobalFileTreeCollapse = viper.GetBool("filetree.collapse-dir")
 }
 
 // initLogging sets up the logging object with a formatter and location

filetree/cache.go

@@ -0,0 +1,74 @@
+package filetree
+
+type TreeCacheKey struct {
+	bottomTreeStart, bottomTreeStop, topTreeStart, topTreeStop int
+}
+
+type TreeCache struct {
+	refTrees []*FileTree
+	cache    map[TreeCacheKey]*FileTree
+}
+
+func (cache *TreeCache) Get(bottomTreeStart, bottomTreeStop, topTreeStart, topTreeStop int) *FileTree {
+	key := TreeCacheKey{bottomTreeStart, bottomTreeStop, topTreeStart, topTreeStop}
+	if value, exists := cache.cache[key]; exists {
+		return value
+	} else {
+
+	}
+	value := cache.buildTree(key)
+	cache.cache[key] = value
+	return value
+}
+
+func (cache *TreeCache) buildTree(key TreeCacheKey) *FileTree {
+	newTree := StackTreeRange(cache.refTrees, key.bottomTreeStart, key.bottomTreeStop)
+
+	for idx := key.topTreeStart; idx <= key.topTreeStop; idx++ {
+		newTree.Compare(cache.refTrees[idx])
+	}
+	return newTree
+}
+
+func (cache *TreeCache) Build() {
+	var bottomTreeStart, bottomTreeStop, topTreeStart, topTreeStop int
+
+	// case 1: layer compare (top tree SIZE is fixed (BUT floats forward), Bottom tree SIZE changes)
+	for selectIdx := 0; selectIdx < len(cache.refTrees); selectIdx++ {
+		bottomTreeStart = 0
+		topTreeStop = selectIdx
+
+		if selectIdx == 0 {
+			bottomTreeStop = selectIdx
+			topTreeStart = selectIdx
+		} else {
+			bottomTreeStop = selectIdx - 1
+			topTreeStart = selectIdx
+		}
+
+		cache.Get(bottomTreeStart, bottomTreeStop, topTreeStart, topTreeStop)
+	}
+
+	// case 2: aggregated compare (bottom tree is ENTIRELY fixed, top tree SIZE changes)
+	for selectIdx := 0; selectIdx < len(cache.refTrees); selectIdx++ {
+		bottomTreeStart = 0
+		topTreeStop = selectIdx
+		if selectIdx == 0 {
+			bottomTreeStop = selectIdx
+			topTreeStart = selectIdx
+		} else {
+			bottomTreeStop = 0
+			topTreeStart = 1
+		}
+
+		cache.Get(bottomTreeStart, bottomTreeStop, topTreeStart, topTreeStop)
+	}
+}
+
+func NewFileTreeCache(refTrees []*FileTree) TreeCache {
+
+	return TreeCache{
+		refTrees: refTrees,
+		cache:    make(map[TreeCacheKey]*FileTree),
+	}
+}

filetree/data.go

@@ -7,7 +7,6 @@ import (
 
 	"github.com/cespare/xxhash"
 	"github.com/sirupsen/logrus"
-	"github.com/spf13/viper"
 )
 
 const (
@@ -17,29 +16,7 @@ const (
 	Removed
 )
 
-// NodeData is the payload for a FileNode
-type NodeData struct {
-	ViewInfo ViewInfo
-	FileInfo FileInfo
-	DiffType DiffType
-}
-
-// ViewInfo contains UI specific detail for a specific FileNode
-type ViewInfo struct {
-	Collapsed bool
-	Hidden    bool
-}
-
-// FileInfo contains tar metadata for a specific FileNode
-type FileInfo struct {
-	Path      string
-	TypeFlag  byte
-	hash      uint64
-	TarHeader tar.Header
-}
-
-// DiffType defines the comparison result between two FileNodes
-type DiffType int
+var GlobalFileTreeCollapse bool
 
 // NewNodeData creates an empty NodeData struct for a FileNode
 func NewNodeData() *NodeData {
@@ -62,7 +39,7 @@ func (data *NodeData) Copy() *NodeData {
 // NewViewInfo creates a default ViewInfo
 func NewViewInfo() (view *ViewInfo) {
 	return &ViewInfo{
-		Collapsed: viper.GetBool("filetree.collapse-dir"),
+		Collapsed: GlobalFileTreeCollapse,
 		Hidden:    false,
 	}
 }
@@ -74,12 +51,10 @@ func (view *ViewInfo) Copy() (newView *ViewInfo) {
 	return newView
 }
 
-var chuckSize = 2 * 1024 * 1024
-
 func getHashFromReader(reader io.Reader) uint64 {
 	h := xxhash.New()
 
-	buf := make([]byte, chuckSize)
+	buf := make([]byte, 1024)
 	for {
 		n, err := reader.Read(buf)
 		if err != nil && err != io.EOF {
@@ -99,20 +74,30 @@ func getHashFromReader(reader io.Reader) uint64 {
 func NewFileInfo(reader *tar.Reader, header *tar.Header, path string) FileInfo {
 	if header.Typeflag == tar.TypeDir {
 		return FileInfo{
-			Path:      path,
-			TypeFlag:  header.Typeflag,
-			hash:      0,
-			TarHeader: *header,
+			Path:     path,
+			TypeFlag: header.Typeflag,
+			Linkname: header.Linkname,
+			hash:     0,
+			Size:     header.FileInfo().Size(),
+			Mode:     header.FileInfo().Mode(),
+			Uid:      header.Uid,
+			Gid:      header.Gid,
+			IsDir:    header.FileInfo().IsDir(),
 		}
 	}
 
 	hash := getHashFromReader(reader)
 
 	return FileInfo{
-		Path:      path,
-		TypeFlag:  header.Typeflag,
-		hash:      hash,
-		TarHeader: *header,
+		Path:     path,
+		TypeFlag: header.Typeflag,
+		Linkname: header.Linkname,
+		hash:     hash,
+		Size:     header.FileInfo().Size(),
+		Mode:     header.FileInfo().Mode(),
+		Uid:      header.Uid,
+		Gid:      header.Gid,
+		IsDir:    header.FileInfo().IsDir(),
 	}
 }
 
@@ -122,10 +107,15 @@ func (data *FileInfo) Copy() *FileInfo {
 		return nil
 	}
 	return &FileInfo{
-		Path:      data.Path,
-		TypeFlag:  data.TypeFlag,
-		hash:      data.hash,
-		TarHeader: data.TarHeader,
+		Path:     data.Path,
+		TypeFlag: data.TypeFlag,
+		Linkname: data.Linkname,
+		hash:     data.hash,
+		Size:     data.Size,
+		Mode:     data.Mode,
+		Uid:      data.Uid,
+		Gid:      data.Gid,
+		IsDir:    data.IsDir,
 	}
 }
 

filetree/data_test.go

@@ -6,7 +6,7 @@ import (
 
 func TestAssignDiffType(t *testing.T) {
 	tree := NewFileTree()
-	node, err := tree.AddPath("/usr", *BlankFileChangeInfo("/usr"))
+	node, _, err := tree.AddPath("/usr", *BlankFileChangeInfo("/usr"))
 	if err != nil {
 		t.Errorf("Expected no error from fetching path. got: %v", err)
 	}

filetree/efficiency.go

@@ -6,17 +6,6 @@ import (
 	"sort"
 )
 
-// EfficiencyData represents the storage and reference statistics for a given file tree path.
-type EfficiencyData struct {
-	Path              string
-	Nodes             []*FileNode
-	CumulativeSize    int64
-	minDiscoveredSize int64
-}
-
-// EfficiencySlice represents an ordered set of EfficiencyData data structures.
-type EfficiencySlice []*EfficiencyData
-
 // Len is required for sorting.
 func (efs EfficiencySlice) Len() int {
 	return len(efs)
@@ -59,19 +48,19 @@ func Efficiency(trees []*FileTree) (float64, EfficiencySlice) {
 
 		if node.IsWhiteout() {
 			sizer := func(curNode *FileNode) error {
-				sizeBytes += curNode.Data.FileInfo.TarHeader.FileInfo().Size()
+				sizeBytes += curNode.Data.FileInfo.Size
 				return nil
 			}
-			stackedTree := StackRange(trees, 0, currentTree-1)
+			stackedTree := StackTreeRange(trees, 0, currentTree-1)
 			previousTreeNode, err := stackedTree.GetNode(node.Path())
 			if err != nil {
 				logrus.Debug(fmt.Sprintf("CurrentTree: %d : %s", currentTree, err))
-			} else if previousTreeNode.Data.FileInfo.TarHeader.FileInfo().IsDir() {
+			} else if previousTreeNode.Data.FileInfo.IsDir {
 				previousTreeNode.VisitDepthChildFirst(sizer, nil)
 			}
 
 		} else {
-			sizeBytes = node.Data.FileInfo.TarHeader.FileInfo().Size()
+			sizeBytes = node.Data.FileInfo.Size
 		}
 
 		data.CumulativeSize += sizeBytes

filetree/efficiency_test.go

@@ -1,7 +1,6 @@
 package filetree
 
 import (
-	"archive/tar"
 	"testing"
 )
 
@@ -11,11 +10,11 @@ func TestEfficencyMap(t *testing.T) {
 		trees[idx] = NewFileTree()
 	}
 
-	trees[0].AddPath("/etc/nginx/nginx.conf", FileInfo{TarHeader: tar.Header{Size: 2000}})
-	trees[0].AddPath("/etc/nginx/public", FileInfo{TarHeader: tar.Header{Size: 3000}})
+	trees[0].AddPath("/etc/nginx/nginx.conf", FileInfo{Size: 2000})
+	trees[0].AddPath("/etc/nginx/public", FileInfo{Size: 3000})
 
-	trees[1].AddPath("/etc/nginx/nginx.conf", FileInfo{TarHeader: tar.Header{Size: 5000}})
-	trees[1].AddPath("/etc/athing", FileInfo{TarHeader: tar.Header{Size: 10000}})
+	trees[1].AddPath("/etc/nginx/nginx.conf", FileInfo{Size: 5000})
+	trees[1].AddPath("/etc/athing", FileInfo{Size: 10000})
 
 	trees[2].AddPath("/etc/.wh.nginx", *BlankFileChangeInfo("/etc/.wh.nginx"))