[18.09] Backport Buildkit fixes for 18.09

builder/builder-next/adapters/containerimage/pull.go

@@ -34,6 +34,7 @@ import (
 	"github.com/moby/buildkit/util/flightcontrol"
 	"github.com/moby/buildkit/util/imageutil"
 	"github.com/moby/buildkit/util/progress"
+	"github.com/moby/buildkit/util/resolver"
 	"github.com/moby/buildkit/util/tracing"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/opencontainers/image-spec/identity"
@@ -51,6 +52,7 @@ type SourceOpt struct {
 	DownloadManager distribution.RootFSDownloadManager
 	MetadataStore   metadata.V2MetadataService
 	ImageStore      image.Store
+	ResolverOpt     resolver.ResolveOptionsFunc
 }
 
 type imageSource struct {
@@ -71,11 +73,16 @@ func (is *imageSource) ID() string {
 	return source.DockerImageScheme
 }
 
-func (is *imageSource) getResolver(ctx context.Context) remotes.Resolver {
-	return docker.NewResolver(docker.ResolverOptions{
+func (is *imageSource) getResolver(ctx context.Context, rfn resolver.ResolveOptionsFunc, ref string) remotes.Resolver {
+	opt := docker.ResolverOptions{
 		Client:      tracing.DefaultClient,
 		Credentials: is.getCredentialsFromSession(ctx),
-	})
+	}
+	if rfn != nil {
+		opt = rfn(ref)
+	}
+	r := docker.NewResolver(opt)
+	return r
 }
 
 func (is *imageSource) getCredentialsFromSession(ctx context.Context) func(string) (string, string, error) {
@@ -118,7 +125,7 @@ func (is *imageSource) resolveRemote(ctx context.Context, ref string, platform *
 		dt   []byte
 	}
 	res, err := is.g.Do(ctx, ref, func(ctx context.Context) (interface{}, error) {
-		dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx), is.ContentStore, platform)
+		dgst, dt, err := imageutil.Config(ctx, ref, is.getResolver(ctx, is.ResolverOpt, ref), is.ContentStore, platform)
 		if err != nil {
 			return nil, err
 		}
@@ -181,7 +188,7 @@ func (is *imageSource) Resolve(ctx context.Context, id source.Identifier) (sourc
 	p := &puller{
 		src:      imageIdentifier,
 		is:       is,
-		resolver: is.getResolver(ctx),
+		resolver: is.getResolver(ctx, is.ResolverOpt, imageIdentifier.Reference.String()),
 		platform: platform,
 	}
 	return p, nil

builder/builder-next/adapters/snapshot/layer.go

@@ -5,10 +5,10 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/boltdb/bolt"
 	"github.com/docker/docker/layer"
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/pkg/errors"
+	bolt "go.etcd.io/bbolt"
 	"golang.org/x/sync/errgroup"
 )
 

builder/builder-next/adapters/snapshot/snapshot.go

@@ -7,7 +7,6 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/boltdb/bolt"
 	"github.com/containerd/containerd/mount"
 	"github.com/containerd/containerd/snapshots"
 	"github.com/docker/docker/daemon/graphdriver"
@@ -16,6 +15,7 @@ import (
 	"github.com/moby/buildkit/snapshot"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
+	bolt "go.etcd.io/bbolt"
 )
 
 var keyParent = []byte("parent")

builder/builder-next/builder.go

@@ -13,16 +13,19 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/builder"
+	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/daemon/images"
 	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/docker/docker/pkg/system"
 	"github.com/docker/libnetwork"
 	controlapi "github.com/moby/buildkit/api/services/control"
+	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/control"
 	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/solver/llbsolver"
 	"github.com/moby/buildkit/util/entitlements"
+	"github.com/moby/buildkit/util/resolver"
 	"github.com/moby/buildkit/util/tracing"
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
@@ -55,6 +58,8 @@ type Opt struct {
 	Dist                images.DistributionServices
 	NetworkController   libnetwork.NetworkController
 	DefaultCgroupParent string
+	ResolverOpt         resolver.ResolveOptionsFunc
+	BuilderConfig       config.BuilderConfig
 }
 
 // Builder can build using BuildKit backend
@@ -132,43 +137,18 @@ func (b *Builder) Prune(ctx context.Context, opts types.BuildCachePruneOptions)
 		return 0, nil, err
 	}
 
-	var unusedFor time.Duration
-	unusedForValues := opts.Filters.Get("unused-for")
-
-	switch len(unusedForValues) {
-	case 0:
-
-	case 1:
-		var err error
-		unusedFor, err = time.ParseDuration(unusedForValues[0])
-		if err != nil {
-			return 0, nil, errors.Wrap(err, "unused-for filter expects a duration (e.g., '24h')")
-		}
-
-	default:
-		return 0, nil, errMultipleFilterValues
-	}
-
-	bkFilter := make([]string, 0, opts.Filters.Len())
-	for cacheField := range cacheFields {
-		values := opts.Filters.Get(cacheField)
-		switch len(values) {
-		case 0:
-			bkFilter = append(bkFilter, cacheField)
-		case 1:
-			bkFilter = append(bkFilter, cacheField+"=="+values[0])
-		default:
-			return 0, nil, errMultipleFilterValues
-		}
+	pi, err := toBuildkitPruneInfo(opts)
+	if err != nil {
+		return 0, nil, err
 	}
 
 	eg.Go(func() error {
 		defer close(ch)
 		return b.controller.Prune(&controlapi.PruneRequest{
-			All:          opts.All,
-			KeepDuration: int64(unusedFor),
-			KeepBytes:    opts.KeepStorage,
-			Filter:       bkFilter,
+			All:          pi.All,
+			KeepDuration: int64(pi.KeepDuration),
+			KeepBytes:    pi.KeepBytes,
+			Filter:       pi.Filter,
 		}, &pruneProxy{
 			streamProxy: streamProxy{ctx: ctx},
 			ch:          ch,
@@ -529,3 +509,41 @@ func toBuildkitExtraHosts(inp []string) (string, error) {
 	}
 	return strings.Join(hosts, ","), nil
 }
+
+func toBuildkitPruneInfo(opts types.BuildCachePruneOptions) (client.PruneInfo, error) {
+	var unusedFor time.Duration
+	unusedForValues := opts.Filters.Get("unused-for")
+
+	switch len(unusedForValues) {
+	case 0:
+
+	case 1:
+		var err error
+		unusedFor, err = time.ParseDuration(unusedForValues[0])
+		if err != nil {
+			return client.PruneInfo{}, errors.Wrap(err, "unused-for filter expects a duration (e.g., '24h')")
+		}
+
+	default:
+		return client.PruneInfo{}, errMultipleFilterValues
+	}
+
+	bkFilter := make([]string, 0, opts.Filters.Len())
+	for cacheField := range cacheFields {
+		values := opts.Filters.Get(cacheField)
+		switch len(values) {
+		case 0:
+			bkFilter = append(bkFilter, cacheField)
+		case 1:
+			bkFilter = append(bkFilter, cacheField+"=="+values[0])
+		default:
+			return client.PruneInfo{}, errMultipleFilterValues
+		}
+	}
+	return client.PruneInfo{
+		All:          opts.All,
+		KeepDuration: unusedFor,
+		KeepBytes:    opts.KeepStorage,
+		Filter:       bkFilter,
+	}, nil
+}

builder/builder-next/controller.go

@@ -6,23 +6,27 @@ import (
 	"path/filepath"
 
 	"github.com/containerd/containerd/content/local"
+	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/builder/builder-next/adapters/containerimage"
 	"github.com/docker/docker/builder/builder-next/adapters/snapshot"
 	containerimageexp "github.com/docker/docker/builder/builder-next/exporter"
 	"github.com/docker/docker/builder/builder-next/imagerefchecker"
 	mobyworker "github.com/docker/docker/builder/builder-next/worker"
+	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/daemon/graphdriver"
+	units "github.com/docker/go-units"
 	"github.com/moby/buildkit/cache"
 	"github.com/moby/buildkit/cache/metadata"
 	registryremotecache "github.com/moby/buildkit/cache/remotecache/registry"
+	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/control"
 	"github.com/moby/buildkit/exporter"
 	"github.com/moby/buildkit/frontend"
 	dockerfile "github.com/moby/buildkit/frontend/dockerfile/builder"
 	"github.com/moby/buildkit/frontend/gateway"
 	"github.com/moby/buildkit/frontend/gateway/forwarder"
 	"github.com/moby/buildkit/snapshot/blobmapping"
-	"github.com/moby/buildkit/solver/boltdbcachestorage"
+	"github.com/moby/buildkit/solver/bboltcachestorage"
 	"github.com/moby/buildkit/worker"
 	"github.com/pkg/errors"
 )
@@ -97,6 +101,7 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
 		MetadataStore:   dist.V2MetadataService,
 		ImageStore:      dist.ImageStore,
 		ReferenceStore:  dist.ReferenceStore,
+		ResolverOpt:     opt.ResolverOpt,
 	})
 	if err != nil {
 		return nil, err
@@ -121,17 +126,23 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
 		return nil, err
 	}
 
-	cacheStorage, err := boltdbcachestorage.NewStore(filepath.Join(opt.Root, "cache.db"))
+	cacheStorage, err := bboltcachestorage.NewStore(filepath.Join(opt.Root, "cache.db"))
 	if err != nil {
 		return nil, err
 	}
 
+	gcPolicy, err := getGCPolicy(opt.BuilderConfig, root)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not get builder GC policy")
+	}
+
 	wopt := mobyworker.Opt{
 		ID:                "moby",
 		SessionManager:    opt.SessionManager,
 		MetadataStore:     md,
 		ContentStore:      store,
 		CacheManager:      cm,
+		GCPolicy:          gcPolicy,
 		Snapshotter:       snapshotter,
 		Executor:          exec,
 		ImageSource:       src,
@@ -160,7 +171,48 @@ func newController(rt http.RoundTripper, opt Opt) (*control.Controller, error) {
 		WorkerController:         wc,
 		Frontends:                frontends,
 		CacheKeyStorage:          cacheStorage,
-		ResolveCacheImporterFunc: registryremotecache.ResolveCacheImporterFunc(opt.SessionManager),
+		ResolveCacheImporterFunc: registryremotecache.ResolveCacheImporterFunc(opt.SessionManager, opt.ResolverOpt),
 		// TODO: set ResolveCacheExporterFunc for exporting cache
 	})
 }
+
+func getGCPolicy(conf config.BuilderConfig, root string) ([]client.PruneInfo, error) {
+	var gcPolicy []client.PruneInfo
+	if conf.GC.Enabled {
+		var (
+			defaultKeepStorage int64
+			err                error
+		)
+
+		if conf.GC.DefaultKeepStorage != "" {
+			defaultKeepStorage, err = units.RAMInBytes(conf.GC.DefaultKeepStorage)
+			if err != nil {
+				return nil, errors.Wrapf(err, "could not parse '%s' as Builder.GC.DefaultKeepStorage config", conf.GC.DefaultKeepStorage)
+			}
+		}
+
+		if conf.GC.Policy == nil {
+			gcPolicy = mobyworker.DefaultGCPolicy(root, defaultKeepStorage)
+		} else {
+			gcPolicy = make([]client.PruneInfo, len(conf.GC.Policy))
+			for i, p := range conf.GC.Policy {
+				b, err := units.RAMInBytes(p.KeepStorage)
+				if err != nil {
+					return nil, err
+				}
+				if b == 0 {
+					b = defaultKeepStorage
+				}
+				gcPolicy[i], err = toBuildkitPruneInfo(types.BuildCachePruneOptions{
+					All:         p.All,
+					KeepStorage: b,
+					Filters:     p.Filter,
+				})
+				if err != nil {
+					return nil, err
+				}
+			}
+		}
+	}
+	return gcPolicy, nil
+}

builder/builder-next/worker/gc.go

@@ -0,0 +1,51 @@
+package worker
+
+import (
+	"math"
+
+	"github.com/moby/buildkit/client"
+)
+
+const defaultCap int64 = 2e9 // 2GB
+
+// tempCachePercent represents the percentage ratio of the cache size in bytes to temporarily keep for a short period of time (couple of days)
+// over the total cache size in bytes. Because there is no perfect value, a mathematically pleasing one was chosen.
+// The value is approximately 13.8
+const tempCachePercent = math.E * math.Pi * math.Phi
+
+// DefaultGCPolicy returns a default builder GC policy
+func DefaultGCPolicy(p string, defaultKeepBytes int64) []client.PruneInfo {
+	keep := defaultKeepBytes
+	if defaultKeepBytes == 0 {
+		keep = detectDefaultGCCap(p)
+	}
+
+	tempCacheKeepBytes := int64(math.Round(float64(keep) / 100. * float64(tempCachePercent)))
+	const minTempCacheKeepBytes = 512 * 1e6 // 512MB
+	if tempCacheKeepBytes < minTempCacheKeepBytes {
+		tempCacheKeepBytes = minTempCacheKeepBytes
+	}
+
+	return []client.PruneInfo{
+		// if build cache uses more than 512MB delete the most easily reproducible data after it has not been used for 2 days
+		{
+			Filter:       []string{"type==source.local,type==exec.cachemount,type==source.git.checkout"},
+			KeepDuration: 48 * 3600, // 48h
+			KeepBytes:    tempCacheKeepBytes,
+		},
+		// remove any data not used for 60 days
+		{
+			KeepDuration: 60 * 24 * 3600, // 60d
+			KeepBytes:    keep,
+		},
+		// keep the unshared build cache under cap
+		{
+			KeepBytes: keep,
+		},
+		// if previous policies were insufficient start deleting internal data to keep build cache under cap
+		{
+			All:       true,
+			KeepBytes: keep,
+		},
+	}
+}

builder/builder-next/worker/gc_unix.go

@@ -0,0 +1,17 @@
+// +build !windows
+
+package worker
+
+import (
+	"syscall"
+)
+
+func detectDefaultGCCap(root string) int64 {
+	var st syscall.Statfs_t
+	if err := syscall.Statfs(root, &st); err != nil {
+		return defaultCap
+	}
+	diskSize := int64(st.Bsize) * int64(st.Blocks) // nolint unconvert
+	avail := diskSize / 10
+	return (avail/(1<<30) + 1) * 1e9 // round up
+}

builder/builder-next/worker/gc_windows.go

@@ -0,0 +1,7 @@
+// +build windows
+
+package worker
+
+func detectDefaultGCCap(root string) int64 {
+	return defaultCap
+}