diff --git a/pkg/executor/build.go b/pkg/executor/build.go index d0a7d932d1..1cb60af78a 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -17,9 +17,7 @@ limitations under the License. package executor import ( - "bytes" "fmt" - "io" "io/ioutil" "os" "path/filepath" @@ -160,30 +158,37 @@ func (s *stageBuilder) build() error { return err } files = command.FilesToSnapshot() - var contents []byte if !s.shouldTakeSnapshot(index, files) { continue } + f, err := ioutil.TempFile("", "") + if err != nil { + return err + } + defer f.Close() + if files == nil || s.opts.SingleSnapshot { - contents, err = s.snapshotter.TakeSnapshotFS() + if err := s.snapshotter.TakeSnapshotFS(f); err != nil { + return err + } } else { // Volumes are very weird. They get created in their command, but snapshotted in the next one. // Add them to the list of files to snapshot. for v := range s.cf.Config.Volumes { files = append(files, v) } - contents, err = s.snapshotter.TakeSnapshot(files) - } - if err != nil { - return err + if err := s.snapshotter.TakeSnapshot(files, f); err != nil { + return err + } } + ck, err := compositeKey.Hash() if err != nil { return err } - if err := s.saveSnapshot(command.String(), ck, contents); err != nil { + if err := s.saveSnapshot(command.String(), ck, f.Name()); err != nil { return err } } @@ -215,16 +220,17 @@ func (s *stageBuilder) shouldTakeSnapshot(index int, files []string) bool { return true } -func (s *stageBuilder) saveSnapshot(createdBy string, ck string, contents []byte) error { - if contents == nil { +func (s *stageBuilder) saveSnapshot(createdBy string, ck string, tarPath string) error { + fi, err := os.Stat(tarPath) + if err != nil { + return err + } + if fi.Size() == 0 { logrus.Info("No files were changed, appending empty layer to config. No layer added to image.") return nil } - // Append the layer to the image - opener := func() (io.ReadCloser, error) { - return ioutil.NopCloser(bytes.NewReader(contents)), nil - } - layer, err := tarball.LayerFromOpener(opener) + + layer, err := tarball.LayerFromFile(tarPath) if err != nil { return err } @@ -306,7 +312,7 @@ func extractImageToDependecyDir(index int, image v1.Image) error { if err := os.MkdirAll(dependencyDir, 0755); err != nil { return err } - logrus.Infof("trying to extract to %s", dependencyDir) + logrus.Debugf("trying to extract to %s", dependencyDir) _, err := util.GetFSFromImage(dependencyDir, image) return err } diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go index 4175d47610..be79a9461f 100644 --- a/pkg/snapshot/snapshot.go +++ b/pkg/snapshot/snapshot.go @@ -17,7 +17,6 @@ limitations under the License. package snapshot import ( - "bytes" "fmt" "io" "io/ioutil" @@ -55,32 +54,16 @@ func (s *Snapshotter) Key() (string, error) { // TakeSnapshot takes a snapshot of the specified files, avoiding directories in the whitelist, and creates // a tarball of the changed files. Return contents of the tarball, and whether or not any files were changed -func (s *Snapshotter) TakeSnapshot(files []string) ([]byte, error) { - buf := bytes.NewBuffer([]byte{}) - filesAdded, err := s.snapshotFiles(buf, files) - if err != nil { - return nil, err - } - contents := buf.Bytes() - if !filesAdded { - return nil, nil - } - return contents, err +func (s *Snapshotter) TakeSnapshot(files []string, w io.Writer) error { + _, err := s.snapshotFiles(w, files) + return err } // TakeSnapshotFS takes a snapshot of the filesystem, avoiding directories in the whitelist, and creates -// a tarball of the changed files. Return contents of the tarball, and whether or not any files were changed -func (s *Snapshotter) TakeSnapshotFS() ([]byte, error) { - buf := bytes.NewBuffer([]byte{}) - filesAdded, err := s.snapShotFS(buf) - if err != nil { - return nil, err - } - contents := buf.Bytes() - if !filesAdded { - return nil, nil - } - return contents, err +// a tarball of the changed files. +func (s *Snapshotter) TakeSnapshotFS(w io.Writer) error { + _, err := s.snapShotFS(w) + return err } // snapshotFiles creates a snapshot (tar) and adds the specified files. diff --git a/pkg/snapshot/snapshot_test.go b/pkg/snapshot/snapshot_test.go index 95daa88f63..9745669a3f 100644 --- a/pkg/snapshot/snapshot_test.go +++ b/pkg/snapshot/snapshot_test.go @@ -45,10 +45,11 @@ func TestSnapshotFSFileChange(t *testing.T) { t.Fatalf("Error setting up fs: %s", err) } // Take another snapshot - contents, err := snapshotter.TakeSnapshotFS() - if err != nil { + b := bytes.Buffer{} + if err := snapshotter.TakeSnapshotFS(&b); err != nil { t.Fatalf("Error taking snapshot of fs: %s", err) } + contents := b.Bytes() if contents == nil { t.Fatal("No files added to snapshot.") } @@ -93,10 +94,11 @@ func TestSnapshotFSChangePermissions(t *testing.T) { t.Fatalf("Error changing permissions on %s: %v", batPath, err) } // Take another snapshot - contents, err := snapshotter.TakeSnapshotFS() - if err != nil { + b := bytes.Buffer{} + if err := snapshotter.TakeSnapshotFS(&b); err != nil { t.Fatalf("Error taking snapshot of fs: %s", err) } + contents := b.Bytes() if contents == nil { t.Fatal("No files added to snapshot.") } @@ -142,12 +144,13 @@ func TestSnapshotFiles(t *testing.T) { filesToSnapshot := []string{ filepath.Join(testDir, "foo"), } - contents, err := snapshotter.TakeSnapshot(filesToSnapshot) - if err != nil { + b := bytes.Buffer{} + if err := snapshotter.TakeSnapshot(filesToSnapshot, &b); err != nil { t.Fatal(err) } expectedFiles := []string{"/", "/tmp", filepath.Join(testDir, "foo")} + contents := b.Bytes() // Check contents of the snapshot, make sure contents is equivalent to snapshotFiles reader := bytes.NewReader(contents) tr := tar.NewReader(reader) @@ -171,11 +174,12 @@ func TestEmptySnapshotFS(t *testing.T) { if err != nil { t.Fatal(err) } + b := bytes.Buffer{} // Take snapshot with no changes - contents, err := snapshotter.TakeSnapshotFS() - if err != nil { + if err := snapshotter.TakeSnapshotFS(&b); err != nil { t.Fatalf("Error taking snapshot of fs: %s", err) } + contents := b.Bytes() // Since we took a snapshot with no changes, contents should be nil if contents != nil { t.Fatal("Files added even though no changes to file system were made.") diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index ef16ceb576..fd53ee7bf1 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -74,7 +74,7 @@ func GetFSFromImage(root string, img v1.Image) ([]string, error) { extractedFiles := []string{} for i, l := range layers { - logrus.Infof("Extracting layer %d", i) + logrus.Debugf("Extracting layer %d", i) r, err := l.Uncompressed() if err != nil { return nil, err