Introduce KOCACHE

pkg/build/cache.go

@@ -0,0 +1,210 @@
+// Copyright 2021 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/partial"
+)
+
+type diffIDToDescriptor map[v1.Hash]v1.Descriptor
+type buildIDToDiffID map[string]v1.Hash
+
+type layerCache struct {
+	buildToDiff map[string]buildIDToDiffID
+	diffToDesc  map[string]diffIDToDescriptor
+	sync.Mutex
+}
+
+type layerFactory func() (v1.Layer, error)
+
+func (c *layerCache) get(ctx context.Context, file string, miss layerFactory) (v1.Layer, error) {
+	if os.Getenv("KOCACHE") == "" {
+		return miss()
+	}
+
+	// Cache hit.
+	if diffid, desc, err := c.getMeta(ctx, file); err == nil {
+		return &lazyLayer{
+			diffid:     *diffid,
+			desc:       *desc,
+			buildLayer: miss,
+		}, nil
+	}
+
+	// Cache miss.
+	layer, err := miss()
+	if err != nil {
+		return nil, err
+	}
+	if err := c.put(ctx, file, layer); err != nil {
+		log.Printf("failed to cache metadata %s: %v", file, err)
+	}
+	return layer, nil
+}
+
+func (c *layerCache) getMeta(ctx context.Context, file string) (*v1.Hash, *v1.Descriptor, error) {
+	buildid, err := getBuildID(ctx, file)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if buildid == "" {
+		return nil, nil, fmt.Errorf("no buildid for %s", file)
+	}
+
+	btod, err := c.readBuildToDiff(file)
+	if err != nil {
+		return nil, nil, err
+	}
+	dtod, err := c.readDiffToDesc(file)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	diffid, ok := btod[buildid]
+	if !ok {
+		return nil, nil, fmt.Errorf("no diffid for %q", buildid)
+	}
+
+	desc, ok := dtod[diffid]
+	if !ok {
+		return nil, nil, fmt.Errorf("no desc for %q", diffid)
+	}
+
+	return &diffid, &desc, nil
+}
+
+// Compute new layer metadata and cache it in-mem and on-disk.
+func (c *layerCache) put(ctx context.Context, file string, layer v1.Layer) error {
+	buildid, err := getBuildID(ctx, file)
+	if err != nil {
+		return err
+	}
+
+	desc, err := partial.Descriptor(layer)
+	if err != nil {
+		return err
+	}
+
+	diffid, err := layer.DiffID()
+	if err != nil {
+		return err
+	}
+
+	btod, ok := c.buildToDiff[file]
+	if !ok {
+		btod = buildIDToDiffID{}
+	}
+	btod[buildid] = diffid
+
+	dtod, ok := c.diffToDesc[file]
+	if !ok {
+		dtod = diffIDToDescriptor{}
+	}
+	dtod[diffid] = *desc
+
+	// TODO: Implement better per-file locking.
+	c.Lock()
+	defer c.Unlock()
+
+	btodf, err := os.OpenFile(filepath.Join(filepath.Dir(file), "buildid-to-diffid"), os.O_RDWR|os.O_CREATE, 0755)
+	if err != nil {
+		return err
+	}
+	defer btodf.Close()
+
+	dtodf, err := os.OpenFile(filepath.Join(filepath.Dir(file), "diffid-to-descriptor"), os.O_RDWR|os.O_CREATE, 0755)
+	if err != nil {
+		return err
+	}
+	defer dtodf.Close()
+
+	enc := json.NewEncoder(btodf)
+	enc.SetIndent("", "  ")
+	if err := enc.Encode(&btod); err != nil {
+		return err
+	}
+
+	enc = json.NewEncoder(dtodf)
+	enc.SetIndent("", "  ")
+	if err := enc.Encode(&dtod); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *layerCache) readDiffToDesc(file string) (diffIDToDescriptor, error) {
+	if dtod, ok := c.diffToDesc[file]; ok {
+		return dtod, nil
+	}
+
+	dtodf, err := os.Open(filepath.Join(filepath.Dir(file), "diffid-to-descriptor"))
+	if err != nil {
+		return nil, err
+	}
+	defer dtodf.Close()
+
+	var dtod diffIDToDescriptor
+	if err := json.NewDecoder(dtodf).Decode(&dtod); err != nil {
+		return nil, err
+	}
+	c.diffToDesc[file] = dtod
+	return dtod, nil
+}
+
+func (c *layerCache) readBuildToDiff(file string) (buildIDToDiffID, error) {
+	if btod, ok := c.buildToDiff[file]; ok {
+		return btod, nil
+	}
+
+	btodf, err := os.Open(filepath.Join(filepath.Dir(file), "buildid-to-diffid"))
+	if err != nil {
+		return nil, err
+	}
+	defer btodf.Close()
+
+	var btod buildIDToDiffID
+	if err := json.NewDecoder(btodf).Decode(&btod); err != nil {
+		return nil, err
+	}
+	c.buildToDiff[file] = btod
+	return btod, nil
+}
+
+func getBuildID(ctx context.Context, file string) (string, error) {
+	cmd := exec.CommandContext(ctx, "go", "tool", "buildid", file)
+	var output bytes.Buffer
+	cmd.Stderr = &output
+	cmd.Stdout = &output
+
+	if err := cmd.Run(); err != nil {
+		log.Printf("Unexpected error running \"go tool buildid %s\": %v\n%v", err, file, output.String())
+		return "", err
+	}
+	return strings.TrimSpace(output.String()), nil
+}

pkg/build/gobuild.go

@@ -78,6 +78,8 @@ type gobuild struct {
 	platformMatcher      *platformMatcher
 	dir                  string
 	labels               map[string]string
+
+	cache *layerCache
 }
 
 // Option is a functional option for NewGo.
@@ -117,6 +119,10 @@ func (gbo *gobuildOpener) Open() (Interface, error) {
 		labels:               gbo.labels,
 		dir:                  gbo.dir,
 		platformMatcher:      matcher,
+		cache: &layerCache{
+			buildToDiff: map[string]buildIDToDiffID{},
+			diffToDesc:  map[string]diffIDToDescriptor{},
+		},
 	}, nil
 }
 
@@ -222,12 +228,6 @@ func platformToString(p v1.Platform) string {
 }
 
 func build(ctx context.Context, ip string, dir string, platform v1.Platform, config Config) (string, error) {
-	tmpDir, err := ioutil.TempDir("", "ko")
-	if err != nil {
-		return "", err
-	}
-	file := filepath.Join(tmpDir, "out")
-
 	buildArgs, err := createBuildArgs(config)
 	if err != nil {
 		return "", err
@@ -236,15 +236,31 @@ func build(ctx context.Context, ip string, dir string, platform v1.Platform, con
 	args := make([]string, 0, 4+len(buildArgs))
 	args = append(args, "build")
 	args = append(args, buildArgs...)
-	args = append(args, "-o", file)
-	args = append(args, ip)
-	cmd := exec.CommandContext(ctx, "go", args...)
-	cmd.Dir = dir
 
 	env, err := buildEnv(platform, os.Environ(), config.Env)
 	if err != nil {
 		return "", fmt.Errorf("could not create env for %s: %w", ip, err)
 	}
+
+	tmpDir, err := ioutil.TempDir("", "ko")
+	if err != nil {
+		return "", err
+	}
+
+	if dir := os.Getenv("KOCACHE"); dir != "" {
+		// TODO(#264): if KOCACHE is unset, default to filepath.Join(os.TempDir(), "ko").
+		tmpDir = filepath.Join(dir, "bin", ip, platformToString(platform))
+		if err := os.MkdirAll(tmpDir, os.ModePerm); err != nil {
+			return "", err
+		}
+	}
+
+	file := filepath.Join(tmpDir, "out")
+
+	args = append(args, "-o", file)
+	args = append(args, ip)
+	cmd := exec.CommandContext(ctx, "go", args...)
+	cmd.Dir = dir
 	cmd.Env = env
 
 	var output bytes.Buffer
@@ -253,7 +269,9 @@ func build(ctx context.Context, ip string, dir string, platform v1.Platform, con
 
 	log.Printf("Building %s for %s", ip, platformToString(platform))
 	if err := cmd.Run(); err != nil {
-		os.RemoveAll(tmpDir)
+		if os.Getenv("KOCACHE") == "" {
+			os.RemoveAll(tmpDir)
+		}
 		log.Printf("Unexpected error running \"go build\": %v\n%v", err, output.String())
 		return "", err
 	}
@@ -643,7 +661,9 @@ func (g *gobuild) buildOne(ctx context.Context, refStr string, base v1.Image, pl
 	if err != nil {
 		return nil, err
 	}
-	defer os.RemoveAll(filepath.Dir(file))
+	if os.Getenv("KOCACHE") == "" {
+		defer os.RemoveAll(filepath.Dir(file))
+	}
 
 	var layers []mutate.Addendum
 
@@ -671,21 +691,15 @@ func (g *gobuild) buildOne(ctx context.Context, refStr string, base v1.Image, pl
 	appDir := "/ko-app"
 	appPath := path.Join(appDir, appFilename(ref.Path()))
 
-	// Construct a tarball with the binary and produce a layer.
-	binaryLayerBuf, err := tarBinary(appPath, file, v1.Time{}, platform)
-	if err != nil {
-		return nil, err
+	miss := func() (v1.Layer, error) {
+		return buildLayer(appPath, file, platform)
 	}
-	binaryLayerBytes := binaryLayerBuf.Bytes()
-	binaryLayer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) {
-		return ioutil.NopCloser(bytes.NewBuffer(binaryLayerBytes)), nil
-	}, tarball.WithCompressedCaching, tarball.WithEstargzOptions(estargz.WithPrioritizedFiles([]string{
-		// When using estargz, prioritize downloading the binary entrypoint.
-		appPath,
-	})))
+
+	binaryLayer, err := g.cache.get(ctx, file, miss)
 	if err != nil {
 		return nil, err
 	}
+
 	layers = append(layers, mutate.Addendum{
 		Layer: binaryLayer,
 		History: v1.History{
@@ -759,6 +773,21 @@ func (g *gobuild) buildOne(ctx context.Context, refStr string, base v1.Image, pl
 	return si, nil
 }
 
+func buildLayer(appPath, file string, platform *v1.Platform) (v1.Layer, error) {
+	// Construct a tarball with the binary and produce a layer.
+	binaryLayerBuf, err := tarBinary(appPath, file, v1.Time{}, platform)
+	if err != nil {
+		return nil, err
+	}
+	binaryLayerBytes := binaryLayerBuf.Bytes()
+	return tarball.LayerFromOpener(func() (io.ReadCloser, error) {
+		return ioutil.NopCloser(bytes.NewBuffer(binaryLayerBytes)), nil
+	}, tarball.WithCompressedCaching, tarball.WithEstargzOptions(estargz.WithPrioritizedFiles([]string{
+		// When using estargz, prioritize downloading the binary entrypoint.
+		appPath,
+	})))
+}
+
 // Append appPath to the PATH environment variable, if it exists. Otherwise,
 // set the PATH environment variable to appPath.
 func updatePath(cf *v1.ConfigFile, appPath string) {