Add buffering for large layers.

pkg/executor/build.go

@@ -17,9 +17,7 @@ limitations under the License.
 package executor
 
 import (
-	"bytes"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -160,30 +158,37 @@ func (s *stageBuilder) build() error {
 			return err
 		}
 		files = command.FilesToSnapshot()
-		var contents []byte
 
 		if !s.shouldTakeSnapshot(index, files) {
 			continue
 		}
 
+		f, err := ioutil.TempFile("", "")
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+
 		if files == nil || s.opts.SingleSnapshot {
-			contents, err = s.snapshotter.TakeSnapshotFS()
+			if err := s.snapshotter.TakeSnapshotFS(f); err != nil {
+				return err
+			}
 		} else {
 			// Volumes are very weird. They get created in their command, but snapshotted in the next one.
 			// Add them to the list of files to snapshot.
 			for v := range s.cf.Config.Volumes {
 				files = append(files, v)
 			}
-			contents, err = s.snapshotter.TakeSnapshot(files)
-		}
-		if err != nil {
-			return err
+			if err := s.snapshotter.TakeSnapshot(files, f); err != nil {
+				return err
+			}
 		}
+
 		ck, err := compositeKey.Hash()
 		if err != nil {
 			return err
 		}
-		if err := s.saveSnapshot(command.String(), ck, contents); err != nil {
+		if err := s.saveSnapshot(command.String(), ck, f.Name()); err != nil {
 			return err
 		}
 	}
@@ -215,16 +220,17 @@ func (s *stageBuilder) shouldTakeSnapshot(index int, files []string) bool {
 	return true
 }
 
-func (s *stageBuilder) saveSnapshot(createdBy string, ck string, contents []byte) error {
-	if contents == nil {
+func (s *stageBuilder) saveSnapshot(createdBy string, ck string, tarPath string) error {
+	fi, err := os.Stat(tarPath)
+	if err != nil {
+		return err
+	}
+	if fi.Size() == 0 {
 		logrus.Info("No files were changed, appending empty layer to config. No layer added to image.")
 		return nil
 	}
-	// Append the layer to the image
-	opener := func() (io.ReadCloser, error) {
-		return ioutil.NopCloser(bytes.NewReader(contents)), nil
-	}
-	layer, err := tarball.LayerFromOpener(opener)
+
+	layer, err := tarball.LayerFromFile(tarPath)
 	if err != nil {
 		return err
 	}
@@ -306,7 +312,7 @@ func extractImageToDependecyDir(index int, image v1.Image) error {
 	if err := os.MkdirAll(dependencyDir, 0755); err != nil {
 		return err
 	}
-	logrus.Infof("trying to extract to %s", dependencyDir)
+	logrus.Debugf("trying to extract to %s", dependencyDir)
 	_, err := util.GetFSFromImage(dependencyDir, image)
 	return err
 }

pkg/snapshot/snapshot.go

@@ -17,7 +17,6 @@ limitations under the License.
 package snapshot
 
 import (
-	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -55,32 +54,16 @@ func (s *Snapshotter) Key() (string, error) {
 
 // TakeSnapshot takes a snapshot of the specified files, avoiding directories in the whitelist, and creates
 // a tarball of the changed files. Return contents of the tarball, and whether or not any files were changed
-func (s *Snapshotter) TakeSnapshot(files []string) ([]byte, error) {
-	buf := bytes.NewBuffer([]byte{})
-	filesAdded, err := s.snapshotFiles(buf, files)
-	if err != nil {
-		return nil, err
-	}
-	contents := buf.Bytes()
-	if !filesAdded {
-		return nil, nil
-	}
-	return contents, err
+func (s *Snapshotter) TakeSnapshot(files []string, w io.Writer) error {
+	_, err := s.snapshotFiles(w, files)
+	return err
 }
 
 // TakeSnapshotFS takes a snapshot of the filesystem, avoiding directories in the whitelist, and creates
-// a tarball of the changed files. Return contents of the tarball, and whether or not any files were changed
-func (s *Snapshotter) TakeSnapshotFS() ([]byte, error) {
-	buf := bytes.NewBuffer([]byte{})
-	filesAdded, err := s.snapShotFS(buf)
-	if err != nil {
-		return nil, err
-	}
-	contents := buf.Bytes()
-	if !filesAdded {
-		return nil, nil
-	}
-	return contents, err
+// a tarball of the changed files.
+func (s *Snapshotter) TakeSnapshotFS(w io.Writer) error {
+	_, err := s.snapShotFS(w)
+	return err
 }
 
 // snapshotFiles creates a snapshot (tar) and adds the specified files.

pkg/snapshot/snapshot_test.go

@@ -45,10 +45,11 @@ func TestSnapshotFSFileChange(t *testing.T) {
 		t.Fatalf("Error setting up fs: %s", err)
 	}
 	// Take another snapshot
-	contents, err := snapshotter.TakeSnapshotFS()
-	if err != nil {
+	b := bytes.Buffer{}
+	if err := snapshotter.TakeSnapshotFS(&b); err != nil {
 		t.Fatalf("Error taking snapshot of fs: %s", err)
 	}
+	contents := b.Bytes()
 	if contents == nil {
 		t.Fatal("No files added to snapshot.")
 	}
@@ -93,10 +94,11 @@ func TestSnapshotFSChangePermissions(t *testing.T) {
 		t.Fatalf("Error changing permissions on %s: %v", batPath, err)
 	}
 	// Take another snapshot
-	contents, err := snapshotter.TakeSnapshotFS()
-	if err != nil {
+	b := bytes.Buffer{}
+	if err := snapshotter.TakeSnapshotFS(&b); err != nil {
 		t.Fatalf("Error taking snapshot of fs: %s", err)
 	}
+	contents := b.Bytes()
 	if contents == nil {
 		t.Fatal("No files added to snapshot.")
 	}
@@ -142,12 +144,13 @@ func TestSnapshotFiles(t *testing.T) {
 	filesToSnapshot := []string{
 		filepath.Join(testDir, "foo"),
 	}
-	contents, err := snapshotter.TakeSnapshot(filesToSnapshot)
-	if err != nil {
+	b := bytes.Buffer{}
+	if err := snapshotter.TakeSnapshot(filesToSnapshot, &b); err != nil {
 		t.Fatal(err)
 	}
 	expectedFiles := []string{"/", "/tmp", filepath.Join(testDir, "foo")}
 
+	contents := b.Bytes()
 	// Check contents of the snapshot, make sure contents is equivalent to snapshotFiles
 	reader := bytes.NewReader(contents)
 	tr := tar.NewReader(reader)
@@ -171,11 +174,12 @@ func TestEmptySnapshotFS(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	b := bytes.Buffer{}
 	// Take snapshot with no changes
-	contents, err := snapshotter.TakeSnapshotFS()
-	if err != nil {
+	if err := snapshotter.TakeSnapshotFS(&b); err != nil {
 		t.Fatalf("Error taking snapshot of fs: %s", err)
 	}
+	contents := b.Bytes()
 	// Since we took a snapshot with no changes, contents should be nil
 	if contents != nil {
 		t.Fatal("Files added even though no changes to file system were made.")

pkg/util/fs_util.go

@@ -74,7 +74,7 @@ func GetFSFromImage(root string, img v1.Image) ([]string, error) {
 	extractedFiles := []string{}
 
 	for i, l := range layers {
-		logrus.Infof("Extracting layer %d", i)
+		logrus.Debugf("Extracting layer %d", i)
 		r, err := l.Uncompressed()
 		if err != nil {
 			return nil, err