progressui: adds a json output that shows raw events for the solver s…

…tatus

This adds an additional display output for the progress indicator to
support a json output. It refactors the progressui package a bit to add
a new method that takes in a `SolveStatusDisplay`. This
`SolveStatusDisplay` can be created by the user using `NewDisplay` with
the various modes as input parameters.

The json output will print the status updates and log messages for the
vertexes that have had updates. Each JSON blob has the same format and
it matches closely with the raw underlying status updates. The display
will still attempt to regulate the number of messages that come through
and will batch various events together if events are sent too quickly
according to the default parameters for rate limiting set in the
progressui package.

At the moment, there is no public API for creating your own
`SolveStatusDisplay` and each of the display implementations borrow
heavily from each other. In the future, it is possible this API could be
expanded. If you need to create a custom progress bar that requires more
than what this package offers, I'd probably recommend just copying the
package and acting on the `SolveStatus` channel directly.

Signed-off-by: Jonathan A. Sternberg <jonathan.sternberg@docker.com>

cmd/buildctl/build.go

@@ -46,7 +46,7 @@ var buildCommand = cli.Command{
 		},
 		cli.StringFlag{
 			Name:  "progress",
-			Usage: "Set type of progress (auto, plain, tty). Use plain to show container output",
+			Usage: "Set type of progress (auto, plain, tty, json). Use plain to show container output",
 			Value: "auto",
 		},
 		cli.StringFlag{

docs/reference/buildctl.md

@@ -63,7 +63,7 @@ USAGE:
 
 OPTIONS:
    --output value, -o value    Define exports for build result, e.g. --output type=image,name=docker.io/username/image,push=true
-   --progress value            Set type of progress (auto, plain, tty). Use plain to show container output (default: "auto")
+   --progress value            Set type of progress (auto, plain, tty, json). Use plain to show container output (default: "auto")
    --trace value               Path to trace file. Defaults to no tracing.
    --local value               Allow build access to the local directory
    --oci-layout value          Allow build access to the local OCI layout

util/progress/progressui/display.go

@@ -16,6 +16,7 @@ import (
 	"github.com/moby/buildkit/client"
 	"github.com/morikuni/aec"
 	digest "github.com/opencontainers/go-digest"
+	"github.com/pkg/errors"
 	"github.com/tonistiigi/units"
 	"github.com/tonistiigi/vt100"
 	"golang.org/x/time/rate"
@@ -42,22 +43,33 @@ func WithDesc(text string, console string) DisplaySolveStatusOpt {
 	}
 }
 
-func DisplaySolveStatus(ctx context.Context, c console.Console, w io.Writer, ch chan *client.SolveStatus, opts ...DisplaySolveStatusOpt) ([]client.VertexWarning, error) {
-	modeConsole := c != nil
+// SolveStatusDisplay is a display that can show the updates to the solver status.
+type SolveStatusDisplay interface {
+	// init initializes the display with options from Update.
+	init(dsso *displaySolveStatusOpts)
+
+	// update updates the internal trace with the latest solve status message.
+	// It does not update the display.
+	update(ss *client.SolveStatus)
 
+	// display will update the visible display with the latest updates.
+	// If done is true, this will notify the display that this is the final
+	// time it will be updated so it can flush any final messages.
+	display(done bool)
+
+	// warnings returns any warnings with the associated trace.
+	warnings() []client.VertexWarning
+}
+
+// UpdateDisplay will stream updates from the channel and update the display from those updates.
+// This function only returns when the update channel is closed.
+func UpdateDisplay(ctx context.Context, d SolveStatusDisplay, ch chan *client.SolveStatus, opts ...DisplaySolveStatusOpt) ([]client.VertexWarning, error) {
 	dsso := &displaySolveStatusOpts{}
 	for _, opt := range opts {
 		opt(dsso)
 	}
 
-	disp := &display{c: c, phase: dsso.phase, desc: dsso.consoleDesc}
-	printer := &textMux{w: w, desc: dsso.textDesc}
-
-	if disp.phase == "" {
-		disp.phase = "Building"
-	}
-
-	t := newTrace(w, modeConsole)
+	d.init(dsso)
 
 	tickerTimeout := 150 * time.Millisecond
 	displayTimeout := 100 * time.Millisecond
@@ -71,53 +83,152 @@ func DisplaySolveStatus(ctx context.Context, c console.Console, w io.Writer, ch
 
 	var done bool
 	ticker := time.NewTicker(tickerTimeout)
-	// implemented as closure because "ticker" can change
-	defer func() {
-		ticker.Stop()
-	}()
+	defer ticker.Stop()
 
 	displayLimiter := rate.NewLimiter(rate.Every(displayTimeout), 1)
 
-	var height int
-	width, _ := disp.getSize()
 	for {
 		select {
 		case <-ctx.Done():
 			return nil, ctx.Err()
 		case <-ticker.C:
 		case ss, ok := <-ch:
 			if ok {
-				t.update(ss, width)
+				d.update(ss)
 			} else {
 				done = true
 			}
 		}
 
-		if modeConsole {
-			width, height = disp.getSize()
-			if done {
-				disp.print(t.displayInfo(), width, height, true)
-				t.printErrorLogs(c)
-				return t.warnings(), nil
-			} else if displayLimiter.Allow() {
-				ticker.Stop()
-				ticker = time.NewTicker(tickerTimeout)
-				disp.print(t.displayInfo(), width, height, false)
-			}
-		} else {
-			if done || displayLimiter.Allow() {
-				printer.print(t)
-				if done {
-					t.printErrorLogs(w)
-					return t.warnings(), nil
-				}
-				ticker.Stop()
-				ticker = time.NewTicker(tickerTimeout)
-			}
+		if done {
+			d.display(true)
+			return d.warnings(), nil
+		} else if displayLimiter.Allow() {
+			d.display(false)
+			ticker.Reset(tickerTimeout)
 		}
 	}
 }
 
+type consoleDisplay struct {
+	t             *trace
+	disp          *display
+	width, height int
+}
+
+func NewConsoleDisplay(c console.Console) SolveStatusDisplay {
+	return &consoleDisplay{
+		t:    newTrace(c, true),
+		disp: &display{c: c},
+	}
+}
+
+func (d *consoleDisplay) init(dsso *displaySolveStatusOpts) {
+	d.disp.phase = dsso.phase
+	d.disp.desc = dsso.consoleDesc
+	if d.disp.phase == "" {
+		d.disp.phase = "Building"
+	}
+	d.width, d.height = d.disp.getSize()
+}
+
+func (d *consoleDisplay) update(ss *client.SolveStatus) {
+	d.t.update(ss, d.width)
+}
+
+func (d *consoleDisplay) display(done bool) {
+	d.width, d.height = d.disp.getSize()
+	d.disp.print(d.t.displayInfo(), d.width, d.height, done)
+	if done {
+		d.t.printErrorLogs(d.t.w)
+	}
+}
+
+func (d *consoleDisplay) warnings() []client.VertexWarning {
+	return d.t.warnings()
+}
+
+type textDisplay struct {
+	t       *trace
+	printer *textMux
+}
+
+func newTextDisplay(w io.Writer, printer vertexPrinter) SolveStatusDisplay {
+	return &textDisplay{
+		t:       newTrace(w, false),
+		printer: newTextMux(w, printer),
+	}
+}
+
+func newJSONDisplay(w io.Writer) SolveStatusDisplay {
+	return newTextDisplay(w, &jsonPrinter{})
+}
+
+func (d *textDisplay) init(dsso *displaySolveStatusOpts) {
+	d.printer.desc = dsso.textDesc
+}
+
+func (d *textDisplay) update(ss *client.SolveStatus) {
+	d.t.update(ss, 80)
+}
+
+func (d *textDisplay) display(done bool) {
+	d.printer.print(d.t)
+	if done {
+		d.t.printErrorLogs(d.t.w)
+	}
+}
+
+func (d *textDisplay) warnings() []client.VertexWarning {
+	return d.t.warnings()
+}
+
+type DisplayMode string
+
+const (
+	AutoMode  DisplayMode = "auto"
+	TtyMode   DisplayMode = "tty"
+	PlainMode DisplayMode = "plain"
+	JSONMode  DisplayMode = "json"
+)
+
+// NewDisplay constructs a SolveStatusDisplay that outputs to the given console.File with the given DisplayMode.
+//
+// This method will return an error when the DisplayMode is invalid or if TtyMode is used but the console.File
+// does not refer to a tty. AutoMode will choose TtyMode or PlainMode depending on if the output is a tty or not.
+func NewDisplay(out console.File, mode DisplayMode) (SolveStatusDisplay, error) {
+	switch mode {
+	case AutoMode, TtyMode, "":
+		if c, err := console.ConsoleFromFile(out); err == nil {
+			return NewConsoleDisplay(c), nil
+		} else if mode == "tty" {
+			return nil, errors.Wrap(err, "failed to get console")
+		}
+		fallthrough
+	case PlainMode:
+		return newTextDisplay(out, nil), nil
+	case JSONMode:
+		return newJSONDisplay(out), nil
+	default:
+		return nil, errors.Errorf("invalid progress mode %s", mode)
+	}
+}
+
+// DisplaySolveStatus will display solve status updates to the given console or writer.
+// If console.Console is nil, this outputs status updates in the plain text format to the io.Writer.
+//
+// For more control over the display method, it is suggested to use NewDisplay and UpdateDisplay instead
+// of this function.
+func DisplaySolveStatus(ctx context.Context, c console.Console, w io.Writer, ch chan *client.SolveStatus, opts ...DisplaySolveStatusOpt) ([]client.VertexWarning, error) {
+	var d SolveStatusDisplay
+	if c != nil {
+		d = NewConsoleDisplay(c)
+	} else {
+		d = newTextDisplay(w, nil)
+	}
+	return UpdateDisplay(ctx, d, ch, opts...)
+}
+
 const termHeight = 6
 const termPad = 10
 
@@ -164,7 +275,6 @@ type vertex struct {
 	logsOffset    int
 	logsBuffer    *ring.Ring // stores last logs to print them on error
 	prev          *client.Vertex
-	events        []string
 	lastBlockTime *time.Time
 	count         int
 	statusUpdates map[string]struct{}

util/progress/progressui/json.go

@@ -0,0 +1,61 @@
+package progressui
+
+import (
+	"time"
+
+	"github.com/moby/buildkit/solver/pb"
+	digest "github.com/opencontainers/go-digest"
+)
+
+// SolveStatus holds the latest status update for a vertex by its digest.
+type SolveStatus struct {
+	Digest   digest.Digest   `json:"digest"`
+	Vertex   Vertex          `json:"vertex,omitempty"`
+	Statuses []VertexStatus  `json:"statuses,omitempty"`
+	Warnings []VertexWarning `json:"warnings,omitempty"`
+	Logs     []VertexLog     `json:"logs,omitempty"`
+}
+
+// Vertex contains information about the vertex itself.
+// Information will only be included if it has changed since the last time
+// that the information was sent or if it is the first time this vertex has
+// been output.
+type Vertex struct {
+	Name      string     `json:"name,omitempty"`
+	Started   *time.Time `json:"started,omitempty"`
+	Completed *time.Time `json:"completed,omitempty"`
+	Cached    bool       `json:"cached,omitempty"`
+	Canceled  bool       `json:"canceled,omitempty"`
+	Error     string     `json:"error,omitempty"`
+}
+
+// VertexStatus holds the status information for a continuous action performed by this vertex.
+// As an example, downloading a file would result in a VertexStatus being created and updated.
+//
+// For non-progress messages, VertexLog is used instead.
+type VertexStatus struct {
+	ID        string     `json:"id"`
+	Name      string     `json:"name,omitempty"`
+	Total     int64      `json:"total,omitempty"`
+	Current   int64      `json:"current"`
+	Timestamp time.Time  `json:"ts"`
+	Started   *time.Time `json:"start,omitempty"`
+	Completed *time.Time `json:"completed,omitempty"`
+}
+
+// VertexWarning holds a warning issued for a particular vertex in the solve graph.
+type VertexWarning struct {
+	Level      int            `json:"level"`
+	Short      []byte         `json:"short"`
+	Detail     [][]byte       `json:"detail"`
+	URL        string         `json:"url,omitempty"`
+	SourceInfo *pb.SourceInfo `json:"sourceInfo,omitempty"`
+	Range      []*pb.Range    `json:"range,omitempty"`
+}
+
+// VertexLog holds log messages produced by the build. For user output that gets updated
+// over time, VertexStatus is used instead.
+type VertexLog struct {
+	Message []byte `json:"msg"`
+	Partial bool   `json:"partial,omitempty"`
+}