Add build.Limiter (ko-build#79)

* Add build.Limiter

You can limit the number of concurrent builds with -j (a la make).

The default value for this is GOMAXPROCS, which seems reasonable.

pkg/build/limit.go

@@ -0,0 +1,55 @@
+// Copyright 2019 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+	"context"
+
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"golang.org/x/sync/semaphore"
+)
+
+// Limiter composes with another Interface to limit the number of concurrent builds.
+type Limiter struct {
+	Builder   Interface
+	semaphore *semaphore.Weighted
+}
+
+// Limiter implements Interface
+var _ Interface = (*Recorder)(nil)
+
+// IsSupportedReference implements Interface
+func (l *Limiter) IsSupportedReference(ip string) bool {
+	return l.Builder.IsSupportedReference(ip)
+}
+
+// Build implements Interface
+func (l *Limiter) Build(ip string) (v1.Image, error) {
+	// TODO(jonjohnsonjr): Build should take a context.Context.
+	if err := l.semaphore.Acquire(context.TODO(), 1); err != nil {
+		return nil, err
+	}
+	defer l.semaphore.Release(1)
+
+	return l.Builder.Build(ip)
+}
+
+// NewLimiter returns a new builder that only allows n concurrent builds of b.
+func NewLimiter(b Interface, n int) *Limiter {
+	return &Limiter{
+		Builder:   b,
+		semaphore: semaphore.NewWeighted(int64(n)),
+	}
+}

pkg/build/limit_test.go

@@ -0,0 +1,58 @@
+// Copyright 2019 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"golang.org/x/sync/errgroup"
+)
+
+type sleeper struct{}
+
+var _ Interface = (*sleeper)(nil)
+
+// IsSupportedReference implements Interface
+func (r *sleeper) IsSupportedReference(ip string) bool {
+	return true
+}
+
+// Build implements Interface
+func (r *sleeper) Build(ip string) (v1.Image, error) {
+	time.Sleep(50 * time.Millisecond)
+	return nil, nil
+}
+
+func TestLimiter(t *testing.T) {
+	b := NewLimiter(&sleeper{}, 2)
+
+	start := time.Now()
+	g, _ := errgroup.WithContext(context.TODO())
+	for i := 0; i <= 10; i++ {
+		g.Go(func() error {
+			_, _ = b.Build("whatever")
+			return nil
+		})
+	}
+	g.Wait()
+
+	// 50 ms * 10 builds / 2 concurrency = ~250ms
+	if time.Now().Before(start.Add(250 * time.Millisecond)) {
+		t.Fatal("Too many builds")
+	}
+}

pkg/commands/apply.go

@@ -32,9 +32,9 @@ func addApply(topLevel *cobra.Command) {
 	no := &options.NameOptions{}
 	fo := &options.FilenameOptions{}
 	ta := &options.TagsOptions{}
-	do := &options.DebugOptions{}
 	so := &options.SelectorOptions{}
 	sto := &options.StrictOptions{}
+	bo := &options.BuildOptions{}
 	apply := &cobra.Command{
 		Use:   "apply -f FILENAME",
 		Short: "Apply the input files with image references resolved to built/pushed image digests.",
@@ -64,7 +64,7 @@ func addApply(topLevel *cobra.Command) {
   cat config.yaml | ko apply -f -`,
 		Args: cobra.NoArgs,
 		Run: func(cmd *cobra.Command, args []string) {
-			builder, err := makeBuilder(do)
+			builder, err := makeBuilder(bo)
 			if err != nil {
 				log.Fatalf("error creating builder: %v", err)
 			}
@@ -130,9 +130,9 @@ func addApply(topLevel *cobra.Command) {
 	options.AddNamingArgs(apply, no)
 	options.AddFileArg(apply, fo)
 	options.AddTagsArg(apply, ta)
-	options.AddDebugArg(apply, do)
 	options.AddSelectorArg(apply, so)
 	options.AddStrictArg(apply, sto)
+	options.AddBuildOptions(apply, bo)
 
 	// Collect the ko-specific apply flags before registering the kubectl global
 	// flags so that we can ignore them when passing kubectl global flags through

pkg/commands/create.go

@@ -32,9 +32,9 @@ func addCreate(topLevel *cobra.Command) {
 	no := &options.NameOptions{}
 	fo := &options.FilenameOptions{}
 	ta := &options.TagsOptions{}
-	do := &options.DebugOptions{}
 	so := &options.SelectorOptions{}
 	sto := &options.StrictOptions{}
+	bo := &options.BuildOptions{}
 	create := &cobra.Command{
 		Use:   "create -f FILENAME",
 		Short: "Create the input files with image references resolved to built/pushed image digests.",
@@ -64,7 +64,7 @@ func addCreate(topLevel *cobra.Command) {
   cat config.yaml | ko create -f -`,
 		Args: cobra.NoArgs,
 		Run: func(cmd *cobra.Command, args []string) {
-			builder, err := makeBuilder(do)
+			builder, err := makeBuilder(bo)
 			if err != nil {
 				log.Fatalf("error creating builder: %v", err)
 			}
@@ -130,9 +130,9 @@ func addCreate(topLevel *cobra.Command) {
 	options.AddNamingArgs(create, no)
 	options.AddFileArg(create, fo)
 	options.AddTagsArg(create, ta)
-	options.AddDebugArg(create, do)
 	options.AddSelectorArg(create, so)
 	options.AddStrictArg(create, sto)
+	options.AddBuildOptions(create, bo)
 
 	// Collect the ko-specific apply flags before registering the kubectl global
 	// flags so that we can ignore them when passing kubectl global flags through

pkg/commands/options/binary.go

@@ -18,13 +18,13 @@ import (
 	"github.com/spf13/cobra"
 )
 
-// BinaryOptions represents options for the ko binary.
-type BinaryOptions struct {
+// PublishOptions represents options for the ko binary.
+type PublishOptions struct {
 	// Path is the import path of the binary to publish.
 	Path string
 }
 
-func AddImageArg(cmd *cobra.Command, lo *BinaryOptions) {
+func AddImageArg(cmd *cobra.Command, lo *PublishOptions) {
 	cmd.Flags().StringVarP(&lo.Path, "image", "i", lo.Path,
 		"The import path of the binary to publish.")
 }

pkg/commands/options/debug.go → pkg/commands/options/build.go

@@ -15,15 +15,20 @@
 package options
 
 import (
+	"runtime"
+
 	"github.com/spf13/cobra"
 )
 
-// DebugOptions holds options to improve debugging containers.
-type DebugOptions struct {
+// BuildOptions represents options for the ko builder.
+type BuildOptions struct {
+	ConcurrentBuilds     int
 	DisableOptimizations bool
 }
 
-func AddDebugArg(cmd *cobra.Command, do *DebugOptions) {
-	cmd.Flags().BoolVar(&do.DisableOptimizations, "disable-optimizations", do.DisableOptimizations,
+func AddBuildOptions(cmd *cobra.Command, bo *BuildOptions) {
+	cmd.Flags().IntVarP(&bo.ConcurrentBuilds, "--jobs", "j", runtime.GOMAXPROCS(0),
+		"The maximum number of concurrent builds")
+	cmd.Flags().BoolVar(&bo.DisableOptimizations, "disable-optimizations", bo.DisableOptimizations,
 		"Disable optimizations when building Go code. Useful when you want to interactively debug the created container.")
 }

pkg/commands/publish.go

@@ -27,7 +27,7 @@ func addPublish(topLevel *cobra.Command) {
 	lo := &options.LocalOptions{}
 	no := &options.NameOptions{}
 	ta := &options.TagsOptions{}
-	do := &options.DebugOptions{}
+	bo := &options.BuildOptions{}
 
 	publish := &cobra.Command{
 		Use:   "publish IMPORTPATH...",
@@ -60,7 +60,7 @@ func addPublish(topLevel *cobra.Command) {
   ko publish --local github.com/foo/bar/cmd/baz github.com/foo/bar/cmd/blah`,
 		Args: cobra.MinimumNArgs(1),
 		Run: func(_ *cobra.Command, args []string) {
-			builder, err := makeBuilder(do)
+			builder, err := makeBuilder(bo)
 			if err != nil {
 				log.Fatalf("error creating builder: %v", err)
 			}
@@ -80,6 +80,6 @@ func addPublish(topLevel *cobra.Command) {
 	options.AddLocalArg(publish, lo)
 	options.AddNamingArgs(publish, no)
 	options.AddTagsArg(publish, ta)
-	options.AddDebugArg(publish, do)
+	options.AddBuildOptions(publish, bo)
 	topLevel.AddCommand(publish)
 }

pkg/commands/resolve.go

@@ -28,9 +28,9 @@ func addResolve(topLevel *cobra.Command) {
 	no := &options.NameOptions{}
 	fo := &options.FilenameOptions{}
 	ta := &options.TagsOptions{}
-	do := &options.DebugOptions{}
 	so := &options.SelectorOptions{}
 	sto := &options.StrictOptions{}
+	bo := &options.BuildOptions{}
 
 	resolve := &cobra.Command{
 		Use:   "resolve -f FILENAME",
@@ -58,7 +58,7 @@ func addResolve(topLevel *cobra.Command) {
   ko resolve --local -f config/`,
 		Args: cobra.NoArgs,
 		Run: func(cmd *cobra.Command, args []string) {
-			builder, err := makeBuilder(do)
+			builder, err := makeBuilder(bo)
 			if err != nil {
 				log.Fatalf("error creating builder: %v", err)
 			}
@@ -73,8 +73,8 @@ func addResolve(topLevel *cobra.Command) {
 	options.AddNamingArgs(resolve, no)
 	options.AddFileArg(resolve, fo)
 	options.AddTagsArg(resolve, ta)
-	options.AddDebugArg(resolve, do)
 	options.AddSelectorArg(resolve, so)
 	options.AddStrictArg(resolve, sto)
+	options.AddBuildOptions(resolve, bo)
 	topLevel.AddCommand(resolve)
 }

pkg/commands/resolver.go

@@ -32,7 +32,7 @@ import (
 	"github.com/mattmoor/dep-notify/pkg/graph"
 )
 
-func gobuildOptions(do *options.DebugOptions) ([]build.Option, error) {
+func gobuildOptions(bo *options.BuildOptions) ([]build.Option, error) {
 	creationTime, err := getCreationTime()
 	if err != nil {
 		return nil, err
@@ -43,14 +43,14 @@ func gobuildOptions(do *options.DebugOptions) ([]build.Option, error) {
 	if creationTime != nil {
 		opts = append(opts, build.WithCreationTime(*creationTime))
 	}
-	if do.DisableOptimizations {
+	if bo.DisableOptimizations {
 		opts = append(opts, build.WithDisabledOptimizations())
 	}
 	return opts, nil
 }
 
-func makeBuilder(do *options.DebugOptions) (*build.Caching, error) {
-	opt, err := gobuildOptions(do)
+func makeBuilder(bo *options.BuildOptions) (*build.Caching, error) {
+	opt, err := gobuildOptions(bo)
 	if err != nil {
 		log.Fatalf("error setting up builder options: %v", err)
 	}
@@ -59,6 +59,8 @@ func makeBuilder(do *options.DebugOptions) (*build.Caching, error) {
 		return nil, err
 	}
 
+	innerBuilder = build.NewLimiter(innerBuilder, bo.ConcurrentBuilds)
+
 	// tl;dr Wrap builder in a caching builder.
 	//
 	// The caching builder should on Build calls:

pkg/commands/run.go

@@ -26,10 +26,10 @@ import (
 // addRun augments our CLI surface with run.
 func addRun(topLevel *cobra.Command) {
 	lo := &options.LocalOptions{}
-	bo := &options.BinaryOptions{}
+	po := &options.PublishOptions{}
 	no := &options.NameOptions{}
 	ta := &options.TagsOptions{}
-	do := &options.DebugOptions{}
+	bo := &options.BuildOptions{}
 
 	run := &cobra.Command{
 		Use:   "run NAME --image=IMPORTPATH",
@@ -45,15 +45,15 @@ func addRun(topLevel *cobra.Command) {
   # This supports relative import paths as well.
   ko run foo --image=./cmd/baz`,
 		Run: func(cmd *cobra.Command, args []string) {
-			builder, err := makeBuilder(do)
+			builder, err := makeBuilder(bo)
 			if err != nil {
 				log.Fatalf("error creating builder: %v", err)
 			}
 			publisher, err := makePublisher(no, lo, ta)
 			if err != nil {
 				log.Fatalf("error creating publisher: %v", err)
 			}
-			imgs, err := publishImages([]string{bo.Path}, publisher, builder)
+			imgs, err := publishImages([]string{po.Path}, publisher, builder)
 			if err != nil {
 				log.Fatalf("failed to publish images: %v", err)
 			}
@@ -88,9 +88,9 @@ func addRun(topLevel *cobra.Command) {
 	}
 	options.AddLocalArg(run, lo)
 	options.AddNamingArgs(run, no)
-	options.AddImageArg(run, bo)
+	options.AddImageArg(run, po)
 	options.AddTagsArg(run, ta)
-	options.AddDebugArg(run, do)
+	options.AddBuildOptions(run, bo)
 
 	topLevel.AddCommand(run)
 }