diff --git a/archive/tar.go b/archive/extract.go similarity index 61% rename from archive/tar.go rename to archive/extract.go index c54f4705d..693a04d45 100644 --- a/archive/tar.go +++ b/archive/extract.go @@ -8,71 +8,12 @@ import ( "path/filepath" ) -type PathInfo struct { - Path string - Info os.FileInfo -} - -func WriteFilesToArchive(tw TarWriter, files []PathInfo) error { - for _, file := range files { - if err := AddFileToArchive(tw, file.Path, file.Info); err != nil { - return err - } - } - return nil -} - -func AddFileToArchive(tw TarWriter, path string, fi os.FileInfo) error { - if fi.Mode()&os.ModeSocket != 0 { - return nil - } - var target string - if fi.Mode()&os.ModeSymlink != 0 { - var err error - target, err = os.Readlink(path) - if err != nil { - return err - } - } - header, err := tar.FileInfoHeader(fi, target) - if err != nil { - return err - } - header.Name = path - - if err := tw.WriteHeader(header); err != nil { - return err - } - if fi.Mode().IsRegular() { - f, err := os.Open(path) - if err != nil { - return err - } - defer f.Close() - if _, err := io.Copy(tw, f); err != nil { - return err - } - } - return nil -} - -func AddDirToArchive(tw TarWriter, srcDir string) error { - srcDir = filepath.Clean(srcDir) - - return filepath.Walk(srcDir, func(file string, fi os.FileInfo, err error) error { - if err != nil { - return err - } - return AddFileToArchive(tw, file, fi) - }) -} - type PathMode struct { Path string Mode os.FileMode } -func Untar(tr TarReader) error { +func Extract(tr TarReader) error { // Avoid umask from changing the file permissions in the tar file. umask := setUmask(0) defer setUmask(umask) diff --git a/archive/extract_test.go b/archive/extract_test.go new file mode 100644 index 000000000..204d6d104 --- /dev/null +++ b/archive/extract_test.go @@ -0,0 +1,106 @@ +package archive_test + +import ( + "archive/tar" + "io/ioutil" + "math/rand" + "os" + "path/filepath" + "testing" + "time" + + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + + "github.com/buildpacks/lifecycle/archive" + h "github.com/buildpacks/lifecycle/testhelpers" +) + +func TestArchiveExtract(t *testing.T) { + rand.Seed(time.Now().UTC().UnixNano()) + spec.Run(t, "extract", testExtract, spec.Report(report.Terminal{})) +} + +func testExtract(t *testing.T, when spec.G, it spec.S) { + var tmpDir string + + it.Before(func() { + var err error + tmpDir, err = ioutil.TempDir("", "archive-extract-test") + h.AssertNil(t, err) + }) + + it.After(func() { + h.AssertNil(t, os.RemoveAll(tmpDir)) + }) + + when("#Extract", func() { + var pathModes = []archive.PathMode{ + {"root", os.ModeDir + 0755}, + {"root/readonly", os.ModeDir + 0500}, + {"root/standarddir", os.ModeDir + 0755}, + {"root/standarddir/somefile", 0644}, + {"root/readonly/readonlysub/somefile", 0444}, + {"root/readonly/readonlysub", os.ModeDir + 0500}, + } + + it.After(func() { + // Make all files os.ModePerm so they can all be cleaned up. + for _, pathMode := range pathModes { + extractedFile := filepath.Join(tmpDir, pathMode.Path) + if _, err := os.Stat(extractedFile); err == nil { + if err := os.Chmod(extractedFile, os.ModePerm); err != nil { + h.AssertNil(t, err) + } + } + } + }) + + it("extracts a tar file", func() { + file, err := os.Open(filepath.Join("testdata", "tar-to-dir", "some-archive.tar")) + h.AssertNil(t, err) + defer file.Close() + + tr := archive.NewNormalizingTarReader(tar.NewReader(file)) + tr.PrependDir(tmpDir) + h.AssertNil(t, archive.Extract(tr)) + + for _, pathMode := range pathModes { + extractedFile := filepath.Join(tmpDir, pathMode.Path) + fileInfo, err := os.Stat(extractedFile) + h.AssertNil(t, err) + h.AssertEq(t, fileInfo.Mode(), pathMode.Mode) + } + }) + + it("fails if file exists where directory needs to be created", func() { + _, err := os.Create(filepath.Join(tmpDir, "root")) + h.AssertNil(t, err) + + file, err := os.Open(filepath.Join("testdata", "tar-to-dir", "some-archive.tar")) + h.AssertNil(t, err) + defer file.Close() + tr := archive.NewNormalizingTarReader(tar.NewReader(file)) + tr.PrependDir(tmpDir) + + h.AssertError(t, archive.Extract(tr), "root: not a directory") + }) + + it("doesn't alter permissions of existing folders", func() { + h.AssertNil(t, os.Mkdir(filepath.Join(tmpDir, "root"), 0744)) + // Update permissions in case umask was applied. + h.AssertNil(t, os.Chmod(filepath.Join(tmpDir, "root"), 0744)) + + file, err := os.Open(filepath.Join("testdata", "tar-to-dir", "some-archive.tar")) + h.AssertNil(t, err) + defer file.Close() + tr := archive.NewNormalizingTarReader(tar.NewReader(file)) + tr.PrependDir(tmpDir) + + h.AssertNil(t, archive.Extract(tr)) + fileInfo, err := os.Stat(filepath.Join(tmpDir, "root")) + h.AssertNil(t, err) + h.AssertEq(t, fileInfo.Mode(), os.ModeDir+0744) + }) + }) +} diff --git a/archive/reader.go b/archive/reader.go index fcc08ef96..320b25f17 100644 --- a/archive/reader.go +++ b/archive/reader.go @@ -28,7 +28,7 @@ func (tr *NormalizingTarReader) ExcludePaths(paths []string) { tr.excludedPaths = paths } -func (tr *NormalizingTarReader) ToWindows() { +func (tr *NormalizingTarReader) FromSlash() { tr.headerOpts = append(tr.headerOpts, func(hdr *tar.Header) *tar.Header { hdr.Name = filepath.FromSlash(hdr.Name) return hdr diff --git a/archive/reader_test.go b/archive/reader_test.go new file mode 100644 index 000000000..ecfb51d62 --- /dev/null +++ b/archive/reader_test.go @@ -0,0 +1,99 @@ +package archive_test + +import ( + "archive/tar" + "io" + "math/rand" + "runtime" + "testing" + "time" + + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + + "github.com/buildpacks/lifecycle/archive" + h "github.com/buildpacks/lifecycle/testhelpers" +) + +func TestReader(t *testing.T) { + rand.Seed(time.Now().UTC().UnixNano()) + spec.Run(t, "tar", testNormalizingTarReader, spec.Report(report.Terminal{})) +} + +func testNormalizingTarReader(t *testing.T, when spec.G, it spec.S) { + when("NormalizingTarWriter", func() { + var ( + ftr *fakeTarReader + ntr *archive.NormalizingTarReader + ) + + it.Before(func() { + ftr = &fakeTarReader{} + ntr = archive.NewNormalizingTarReader(ftr) + ftr.pushHeader(&tar.Header{Name: "some/path"}) + }) + + when("#Strip", func() { + it("removes leading dirs", func() { + ntr.Strip("some") + hdr, err := ntr.Next() + h.AssertNil(t, err) + h.AssertEq(t, hdr.Name, "/path") + }) + }) + + when("#FromSlash", func() { + it("converts path separators", func() { + ntr.FromSlash() + hdr, err := ntr.Next() + h.AssertNil(t, err) + if runtime.GOOS == "windows" { + h.AssertEq(t, hdr.Name, `some\path`) + } else { + h.AssertEq(t, hdr.Name, `some/path`) + } + }) + }) + + when("#PrependDir", func() { + it("prepends the dir", func() { + ntr.PrependDir("/super-dir") + hdr, err := ntr.Next() + h.AssertNil(t, err) + h.AssertEq(t, hdr.Name, `/super-dir/some/path`) + }) + }) + + when("#Exclude", func() { + it("skips excluded entries", func() { + ftr.pushHeader(&tar.Header{Name: "excluded-dir"}) + ftr.pushHeader(&tar.Header{Name: "excluded-dir/file"}) + ntr.ExcludePaths([]string{"excluded-dir"}) + hdr, err := ntr.Next() + h.AssertNil(t, err) + h.AssertEq(t, hdr.Name, `some/path`) + }) + }) + }) +} + +type fakeTarReader struct { + hdrs []*tar.Header +} + +func (r *fakeTarReader) Next() (*tar.Header, error) { + if len(r.hdrs) == 0 { + return nil, io.EOF + } + hdr := r.hdrs[0] + r.hdrs = r.hdrs[1:] + return hdr, nil +} + +func (r *fakeTarReader) Read(b []byte) (int, error) { + return len(b), nil +} + +func (r *fakeTarReader) pushHeader(hdr *tar.Header) { + r.hdrs = append([]*tar.Header{hdr}, r.hdrs...) +} diff --git a/archive/write.go b/archive/write.go new file mode 100644 index 000000000..780bdeec6 --- /dev/null +++ b/archive/write.go @@ -0,0 +1,67 @@ +package archive + +import ( + "archive/tar" + "io" + "os" + "path/filepath" +) + +type PathInfo struct { + Path string + Info os.FileInfo +} + +func WriteFilesToArchive(tw TarWriter, files []PathInfo) error { + for _, file := range files { + if err := AddFileToArchive(tw, file.Path, file.Info); err != nil { + return err + } + } + return nil +} + +func AddFileToArchive(tw TarWriter, path string, fi os.FileInfo) error { + if fi.Mode()&os.ModeSocket != 0 { + return nil + } + var target string + if fi.Mode()&os.ModeSymlink != 0 { + var err error + target, err = os.Readlink(path) + if err != nil { + return err + } + } + header, err := tar.FileInfoHeader(fi, target) + if err != nil { + return err + } + header.Name = path + + if err := tw.WriteHeader(header); err != nil { + return err + } + if fi.Mode().IsRegular() { + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + if _, err := io.Copy(tw, f); err != nil { + return err + } + } + return nil +} + +func AddDirToArchive(tw TarWriter, srcDir string) error { + srcDir = filepath.Clean(srcDir) + + return filepath.Walk(srcDir, func(file string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + return AddFileToArchive(tw, file, fi) + }) +} diff --git a/archive/tar_test.go b/archive/write_test.go similarity index 56% rename from archive/tar_test.go rename to archive/write_test.go index 40a7c7974..a596b9b40 100644 --- a/archive/tar_test.go +++ b/archive/write_test.go @@ -18,17 +18,17 @@ import ( h "github.com/buildpacks/lifecycle/testhelpers" ) -func TestTar(t *testing.T) { +func TestArchiveWrite(t *testing.T) { rand.Seed(time.Now().UTC().UnixNano()) - spec.Run(t, "tar", testTar, spec.Report(report.Terminal{})) + spec.Run(t, "tar", testWrite, spec.Report(report.Terminal{})) } -func testTar(t *testing.T, when spec.G, it spec.S) { +func testWrite(t *testing.T, when spec.G, it spec.S) { var tmpDir string it.Before(func() { var err error - tmpDir, err = ioutil.TempDir("", "extract-tar-test") + tmpDir, err = ioutil.TempDir("", "archive-write-test") h.AssertNil(t, err) }) @@ -36,76 +36,6 @@ func testTar(t *testing.T, when spec.G, it spec.S) { h.AssertNil(t, os.RemoveAll(tmpDir)) }) - when("#Untar", func() { - var pathModes = []archive.PathMode{ - {"root", os.ModeDir + 0755}, - {"root/readonly", os.ModeDir + 0500}, - {"root/standarddir", os.ModeDir + 0755}, - {"root/standarddir/somefile", 0644}, - {"root/readonly/readonlysub/somefile", 0444}, - {"root/readonly/readonlysub", os.ModeDir + 0500}, - } - - it.After(func() { - // Make all files os.ModePerm so they can all be cleaned up. - for _, pathMode := range pathModes { - extractedFile := filepath.Join(tmpDir, pathMode.Path) - if _, err := os.Stat(extractedFile); err == nil { - if err := os.Chmod(extractedFile, os.ModePerm); err != nil { - h.AssertNil(t, err) - } - } - } - }) - - it("extracts a tar file", func() { - file, err := os.Open(filepath.Join("testdata", "tar-to-dir", "some-archive.tar")) - h.AssertNil(t, err) - defer file.Close() - - tr := archive.NewNormalizingTarReader(tar.NewReader(file)) - tr.PrependDir(tmpDir) - h.AssertNil(t, archive.Untar(tr)) - - for _, pathMode := range pathModes { - extractedFile := filepath.Join(tmpDir, pathMode.Path) - fileInfo, err := os.Stat(extractedFile) - h.AssertNil(t, err) - h.AssertEq(t, fileInfo.Mode(), pathMode.Mode) - } - }) - - it("fails if file exists where directory needs to be created", func() { - _, err := os.Create(filepath.Join(tmpDir, "root")) - h.AssertNil(t, err) - - file, err := os.Open(filepath.Join("testdata", "tar-to-dir", "some-archive.tar")) - h.AssertNil(t, err) - defer file.Close() - tr := archive.NewNormalizingTarReader(tar.NewReader(file)) - tr.PrependDir(tmpDir) - - h.AssertError(t, archive.Untar(tr), "root: not a directory") - }) - - it("doesn't alter permissions of existing folders", func() { - h.AssertNil(t, os.Mkdir(filepath.Join(tmpDir, "root"), 0744)) - // Update permissions in case umask was applied. - h.AssertNil(t, os.Chmod(filepath.Join(tmpDir, "root"), 0744)) - - file, err := os.Open(filepath.Join("testdata", "tar-to-dir", "some-archive.tar")) - h.AssertNil(t, err) - defer file.Close() - tr := archive.NewNormalizingTarReader(tar.NewReader(file)) - tr.PrependDir(tmpDir) - - h.AssertNil(t, archive.Untar(tr)) - fileInfo, err := os.Stat(filepath.Join(tmpDir, "root")) - h.AssertNil(t, err) - h.AssertEq(t, fileInfo.Mode(), os.ModeDir+0744) - }) - }) - when("#AddDirToArchive", func() { var ( uid = 1234 diff --git a/archive/writer_test.go b/archive/writer_test.go new file mode 100644 index 000000000..746009f20 --- /dev/null +++ b/archive/writer_test.go @@ -0,0 +1,113 @@ +package archive_test + +import ( + "archive/tar" + "math/rand" + "runtime" + "testing" + "time" + + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + + "github.com/buildpacks/lifecycle/archive" + h "github.com/buildpacks/lifecycle/testhelpers" +) + +func TestWriter(t *testing.T) { + rand.Seed(time.Now().UTC().UnixNano()) + spec.Run(t, "tar", testNormalizingTarWriter, spec.Report(report.Terminal{})) +} + +func testNormalizingTarWriter(t *testing.T, when spec.G, it spec.S) { + when("NormalizingTarWriter", func() { + var ( + ftw *fakeTarWriter + ntw *archive.NormalizingTarWriter + ) + + it.Before(func() { + ftw = &fakeTarWriter{} + ntw = archive.NewNormalizingTarWriter(ftw) + }) + + it("normalizes the mod time", func() { + h.AssertNil(t, ntw.WriteHeader(&tar.Header{ + ModTime: time.Now(), + })) + if !ftw.getLastHeader().ModTime.Equal(time.Date(1980, time.January, 1, 0, 0, 1, 0, time.UTC)) { + t.Fatalf("failed to normalize the mod time") + } + }) + + when("#UID", func() { + it("sets the uid", func() { + ntw.WithUID(999) + h.AssertNil(t, ntw.WriteHeader(&tar.Header{ + Uid: 888, + })) + h.AssertEq(t, ftw.getLastHeader().Uid, 999) + }) + }) + + when("#GID", func() { + it("sets the gid", func() { + ntw.WithGID(999) + h.AssertNil(t, ntw.WriteHeader(&tar.Header{ + Gid: 888, + })) + h.AssertEq(t, ftw.getLastHeader().Gid, 999) + }) + }) + + when("#ToPosix", func() { + it.Before(func() { + ntw.ToPosix() + }) + + when("path is posix", func() { + it("does nothing", func() { + h.AssertNil(t, ntw.WriteHeader(&tar.Header{ + Name: "/some/file/path", + })) + h.AssertEq(t, ftw.getLastHeader().Name, "/some/file/path") + }) + }) + + when("path is windows", func() { + it.Before(func() { + if runtime.GOOS != "window" { + t.Skip("windows specific test") + } + }) + + it("converts to posix", func() { + h.AssertNil(t, ntw.WriteHeader(&tar.Header{ + Name: `c:\some\file\path`, + })) + h.AssertEq(t, ftw.getLastHeader().Name, "/some/file/path") + }) + }) + }) + }) +} + +type fakeTarWriter struct { + hdr *tar.Header +} + +func (w *fakeTarWriter) WriteHeader(hdr *tar.Header) error { + w.hdr = hdr + return nil +} +func (w *fakeTarWriter) getLastHeader() *tar.Header { + return w.hdr +} + +func (w *fakeTarWriter) Write(b []byte) (int, error) { + return len(b), nil +} + +func (w *fakeTarWriter) Close() error { + return nil +} diff --git a/layers/extract.go b/layers/extract.go index a0092018c..b760fa8ee 100644 --- a/layers/extract.go +++ b/layers/extract.go @@ -10,7 +10,7 @@ import ( func Extract(r io.Reader, dest string) error { tr := tarReader(r, dest) - return archive.Untar(tr) + return archive.Extract(tr) } func tarReader(r io.Reader, dest string) archive.TarReader { @@ -20,7 +20,7 @@ func tarReader(r io.Reader, dest string) archive.TarReader { if runtime.GOOS == "windows" { tr.ExcludePaths([]string{"Hives"}) tr.Strip(`Files`) - tr.ToWindows() + tr.FromSlash() if dest == "" { dest = "c:" }