Add support for recursively resolving directory symlinks. (ko-build#73)

* Add support for recursively resolving directory symlinks.

This adds support for properly resolving directory symlinks within kodata.
I verified that with this I can symlink `.git/refs` into `kodata/` and (with
changes) resolve the `.git/HEAD` symlink to read the appropriate
`ref: refs/heads/...` files with the commit SHA.

* Incorporate code review feedback

cmd/ko/test/kodata/HEAD

@@ -0,0 +1 @@
+../../../../.git/HEAD

cmd/ko/test/kodata/refs

@@ -0,0 +1 @@
+../../../../.git/refs

cmd/ko/test/main.go

@@ -29,4 +29,11 @@ func main() {
 		log.Fatalf("Error reading %q: %v", file, err)
 	}
 	log.Printf(string(bytes))
+
+	file = filepath.Join(dp, "refs/heads/master")
+	bytes, err = ioutil.ReadFile(file)
+	if err != nil {
+		log.Fatalf("Error reading %q: %v", file, err)
+	}
+	log.Printf(string(bytes))
 }

cmd/ko/test/test.yaml

@@ -20,5 +20,5 @@ metadata:
 spec:
   containers:
   - name: obiwan
-    image: github.com/google/ko/cmd/test
+    image: github.com/google/ko/cmd/ko/test
   restartPolicy: Never

pkg/build/gobuild.go

@@ -276,28 +276,15 @@ func (g *gobuild) kodataPath(s string) (string, error) {
 // Where kodata lives in the image.
 const kodataRoot = "/var/run/ko"
 
-func (g *gobuild) tarKoData(importpath string) (*bytes.Buffer, error) {
-	buf := bytes.NewBuffer(nil)
-	// Compress this before calling tarball.LayerFromOpener, since it eagerly
-	// calculates digests and diffids. This prevents us from double compressing
-	// the layer when we have to actually upload the blob.
-	//
-	// https://github.com/google/go-containerregistry/issues/413
-	gw, _ := gzip.NewWriterLevel(buf, gzip.BestSpeed)
-	defer gw.Close()
-	tw := tar.NewWriter(gw)
-	defer tw.Close()
-
-	root, err := g.kodataPath(importpath)
-	if err != nil {
-		return nil, err
-	}
-
-	err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+// walkRecursive performs a filepath.Walk of the given root directory adding it
+// to the provided tar.Writer with root -> chroot.  All symlinks are dereferenced,
+// which is what leads to recursion when we encounter a directory symlink.
+func walkRecursive(tw *tar.Writer, root, chroot string) error {
+	return filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
 		if path == root {
-			// Add an entry for /var/run/ko
+			// Add an entry for the root directory of our walk.
 			return tw.WriteHeader(&tar.Header{
-				Name:     kodataRoot,
+				Name:     chroot,
 				Typeflag: tar.TypeDir,
 				// Use a fixed Mode, so that this isn't sensitive to the directory and umask
 				// under which it was created. Additionally, windows can only set 0222,
@@ -312,12 +299,22 @@ func (g *gobuild) tarKoData(importpath string) (*bytes.Buffer, error) {
 		if info.Mode().IsDir() {
 			return nil
 		}
+		newPath := filepath.Join(chroot, path[len(root):])
+
+		path, err = filepath.EvalSymlinks(path)
+		if err != nil {
+			return err
+		}
 
 		// Chase symlinks.
 		info, err = os.Stat(path)
 		if err != nil {
 			return err
 		}
+		// Skip other directories.
+		if info.Mode().IsDir() {
+			return walkRecursive(tw, path, newPath)
+		}
 
 		// Open the file to copy it into the tarball.
 		file, err := os.Open(path)
@@ -327,7 +324,6 @@ func (g *gobuild) tarKoData(importpath string) (*bytes.Buffer, error) {
 		defer file.Close()
 
 		// Copy the file into the image tarball.
-		newPath := filepath.Join(kodataRoot, path[len(root):])
 		if err := tw.WriteHeader(&tar.Header{
 			Name:     newPath,
 			Size:     info.Size(),
@@ -342,11 +338,26 @@ func (g *gobuild) tarKoData(importpath string) (*bytes.Buffer, error) {
 		_, err = io.Copy(tw, file)
 		return err
 	})
+}
+
+func (g *gobuild) tarKoData(importpath string) (*bytes.Buffer, error) {
+	buf := bytes.NewBuffer(nil)
+	// Compress this before calling tarball.LayerFromOpener, since it eagerly
+	// calculates digests and diffids. This prevents us from double compressing
+	// the layer when we have to actually upload the blob.
+	//
+	// https://github.com/google/go-containerregistry/issues/413
+	gw, _ := gzip.NewWriterLevel(buf, gzip.BestSpeed)
+	defer gw.Close()
+	tw := tar.NewWriter(gw)
+	defer tw.Close()
+
+	root, err := g.kodataPath(importpath)
 	if err != nil {
 		return nil, err
 	}
 
-	return buf, nil
+	return buf, walkRecursive(tw, root, kodataRoot)
 }
 
 // Build implements build.Interface

pkg/build/gobuild_test.go

@@ -253,19 +253,15 @@ func TestGoBuild(t *testing.T) {
 	})
 
 	t.Run("check app layer contents", func(t *testing.T) {
-		expectedHash := v1.Hash{
-			Algorithm: "sha256",
-			Hex:       "4379f30a6c6f66221c3c54dddd378fcfa5a7304a6655ff783b102069c0f943ab",
-		}
-		appLayer := ls[baseLayers]
+		dataLayer := ls[baseLayers]
 
-		if got, err := appLayer.Digest(); err != nil {
+		if _, err := dataLayer.Digest(); err != nil {
 			t.Errorf("Digest() = %v", err)
-		} else if got != expectedHash {
-			t.Errorf("Digest() = %v, want %v", got, expectedHash)
 		}
+		// We don't check the data layer here because it includes a symlink of refs and
+		// will produce a distinct hash each time we commit something.
 
-		r, err := appLayer.Uncompressed()
+		r, err := dataLayer.Uncompressed()
 		if err != nil {
 			t.Errorf("Uncompressed() = %v", err)
 		}