Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ls: add format, quiet and no-trunc opts #830

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions builder/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,12 @@ package builder

import (
"context"
"encoding/json"
"os"
"sort"
"strings"
"sync"
"time"

"github.com/docker/buildx/driver"
"github.com/docker/buildx/store"
Expand Down Expand Up @@ -240,6 +243,28 @@ func (b *Builder) Factory(ctx context.Context) (_ driver.Factory, err error) {
return b.driverFactory.Factory, err
}

func (b *Builder) MarshalJSON() ([]byte, error) {
var err string
if b.err != nil {
err = strings.TrimSpace(b.err.Error())
}
return json.Marshal(struct {
Name string
Driver string
LastActivity time.Time `json:",omitempty"`
Dynamic bool
Nodes []Node
Err string `json:",omitempty"`
}{
Name: b.Name,
Driver: b.Driver,
LastActivity: b.LastActivity,
Dynamic: b.Dynamic,
Nodes: b.nodes,
Err: err,
})
}

// GetBuilders returns all builders
func GetBuilders(dockerCli command.Cli, txn *store.Txn) ([]*Builder, error) {
storeng, err := txn.List()
Expand Down
42 changes: 42 additions & 0 deletions builder/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@ package builder

import (
"context"
"encoding/json"
"strings"

"github.com/containerd/containerd/platforms"
"github.com/docker/buildx/driver"
ctxkube "github.com/docker/buildx/driver/kubernetes/context"
"github.com/docker/buildx/store"
Expand Down Expand Up @@ -163,6 +166,45 @@ func (b *Builder) LoadNodes(ctx context.Context, withData bool) (_ []Node, err e
return b.nodes, nil
}

func (n *Node) MarshalJSON() ([]byte, error) {
var status string
if n.DriverInfo != nil {
status = n.DriverInfo.Status.String()
}
var err string
if n.Err != nil {
status = "error"
err = strings.TrimSpace(n.Err.Error())
}
var pp []string
for _, p := range n.Platforms {
pp = append(pp, platforms.Format(p))
}
return json.Marshal(struct {
Comment on lines +179 to +183
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was initially wondering if we should have an actual type for this; possibly if we need specific formatting functions for printing, we could define those on the type.

Then I wondered what we needed the ad-hoc type for (was it because of recursion happening during MarshalJSON?); but then noticed the platforms handling above.

Do you know what specifically was needed to use the platforms.Format()? Out of curiousity, I tried what happened if we use []ocispecs.Platform as-is, and (at a glance ⚠️), the output seems to look fine.

This is still using the ad-hoc type, but updating it to use OCI platform;

diff --git a/builder/node.go b/builder/node.go
index 665413f7..2954b153 100644
--- a/builder/node.go
+++ b/builder/node.go
@@ -5,7 +5,6 @@ import (
        "encoding/json"
        "strings"

-       "github.com/containerd/containerd/platforms"
        "github.com/docker/buildx/driver"
        ctxkube "github.com/docker/buildx/driver/kubernetes/context"
        "github.com/docker/buildx/store"
@@ -176,25 +175,21 @@ func (n *Node) MarshalJSON() ([]byte, error) {
                status = "error"
                err = strings.TrimSpace(n.Err.Error())
        }
-       var pp []string
-       for _, p := range n.Platforms {
-               pp = append(pp, platforms.Format(p))
-       }
        return json.Marshal(struct {
                Name        string
                Endpoint    string
-               Platforms   []string          `json:",omitempty"`
-               Flags       []string          `json:",omitempty"`
-               DriverOpts  map[string]string `json:",omitempty"`
-               Files       map[string][]byte `json:",omitempty"`
-               Status      string            `json:",omitempty"`
-               ProxyConfig map[string]string `json:",omitempty"`
-               Version     string            `json:",omitempty"`
-               Err         string            `json:",omitempty"`
+               Platforms   []ocispecs.Platform `json:",omitempty"`
+               Flags       []string            `json:",omitempty"`
+               DriverOpts  map[string]string   `json:",omitempty"`
+               Files       map[string][]byte   `json:",omitempty"`
+               Status      string              `json:",omitempty"`
+               ProxyConfig map[string]string   `json:",omitempty"`
+               Version     string              `json:",omitempty"`
+               Err         string              `json:",omitempty"`
        }{
                Name:        n.Name,
                Endpoint:    n.Endpoint,
-               Platforms:   pp,
+               Platforms:   n.Platforms,
                Flags:       n.Flags,
                DriverOpts:  n.DriverOpts,
                Files:       n.Files,

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

With the above, the output seems to be fine (but perhaps there's specific corner-cases we're not accounting for?)

./bin/build/docker-buildx ls
NAME/NODE           DRIVER/ENDPOINT     STATUS    BUILDKIT                           PLATFORMS
default *           docker
 \_ default          \_ default         running   22.06.0-beta.0-902-g2708be0db4.m   linux/arm64, linux/amd64, linux/amd64/v2, linux/riscv64, linux/ppc64le, linux/s390x,…
desktop-linux       docker
 \_ desktop-linux    \_ desktop-linux   running   22.06.0-beta.0-902-g2708be0db4.m   linux/arm64, linux/amd64, linux/amd64/v2, linux/riscv64, linux/ppc64le, linux/s390x,…
foo                 docker
 \_ foo              \_ foo             error

Failed to get status for foo (foo): Unix socket path "/var/folders/6f/tz5jf4nn1_n5jb0ctrrw5p2w0000gn/T/TestBuildWithBuildercustom_context67217891/001/docker.sock" is too long

👆 😂 Ah, fun; looks like I ran some integration test on my machine, and macOS "temp" directories are symlinks, and the actual location looks to be too long.

I also tried outputing only the .Platforms, which also seems to work, although (not sure where those come from), it looks like there's empty lines in between each row 🤔

 ./bin/build/docker-buildx ls --format '{{.Platforms}}'

linux/arm64, linux/amd64, linux/amd64/v2, linux/riscv64, linux/ppc64le, linux/s390x, linux/mips64le, linux/mips64

linux/arm64, linux/amd64, linux/amd64/v2, linux/riscv64, linux/ppc64le, linux/s390x, linux/mips64le, linux/mips64

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

but perhaps there's specific corner-cases we're not accounting for?

Oh! Perhaps this was for the JSON format? As without it, it would render the platform as a struct? (Wondering if perhaps we should actually keep that 🤔)

 ./bin/build/docker-buildx ls --format json
{"Name":"default","Driver":"docker","Dynamic":false,"Nodes":[{"Name":"default","Endpoint":"default","Platforms":[{"architecture":"arm64","os":"linux"},{"architecture":"amd64","os":"linux"},{"architecture":"amd64","os":"linux","variant":"v2"},{"architecture":"riscv64","os":"linux"},{"architecture":"ppc64le","os":"linux"},{"architecture":"s390x","os":"linux"},{"architecture":"mips64le","os":"linux"},{"architecture":"mips64","os":"linux"}],"Status":"running","Version":"22.06.0-beta.0-902-g2708be0db4.m"}]}
{"Name":"desktop-linux","Driver":"docker","Dynamic":false,"Nodes":[{"Name":"desktop-linux","Endpoint":"desktop-linux","Platforms":[{"architecture":"arm64","os":"linux"},{"architecture":"amd64","os":"linux"},{"architecture":"amd64","os":"linux","variant":"v2"},{"architecture":"riscv64","os":"linux"},{"architecture":"ppc64le","os":"linux"},{"architecture":"s390x","os":"linux"},{"architecture":"mips64le","os":"linux"},{"architecture":"mips64","os":"linux"}],"Status":"running","Version":"22.06.0-beta.0-902-g2708be0db4.m"}]}
{"Name":"foo","Driver":"docker","Dynamic":false,"Nodes":[{"Name":"foo","Endpoint":"foo","Status":"error","Err":"Unix socket path \"/var/folders/6f/tz5jf4nn1_n5jb0ctrrw5p2w0000gn/T/TestBuildWithBuildercustom_context67217891/001/docker.sock\" is too long"}]}

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, and I guess it doesn't show the default / native platform; (linux/arm64* vs linux/arm64);

edit: ⚠️ actually, that's not it (where is that added?)

If we really want a string presentation in all cases, we could have a local type that wraps oci.Platform and implements MarshalText;

diff --git a/builder/node.go b/builder/node.go
index 665413f7..5ee00381 100644
--- a/builder/node.go
+++ b/builder/node.go
@@ -166,6 +166,12 @@ func (b *Builder) LoadNodes(ctx context.Context, withData bool) (_ []Node, err e
        return b.nodes, nil
 }

+type Platform ocispecs.Platform
+
+func (p *Platform) MarshalText() ([]byte, error) {
+       return []byte(platforms.Format(ocispecs.Platform(*p))), nil
+}
+
 func (n *Node) MarshalJSON() ([]byte, error) {
        var status string
        if n.DriverInfo != nil {
@@ -176,14 +182,14 @@ func (n *Node) MarshalJSON() ([]byte, error) {
                status = "error"
                err = strings.TrimSpace(n.Err.Error())
        }
-       var pp []string
+       var pp []Platform
        for _, p := range n.Platforms {
-               pp = append(pp, platforms.Format(p))
+               pp = append(pp, Platform(p))
        }
        return json.Marshal(struct {
                Name        string
                Endpoint    string
-               Platforms   []string          `json:",omitempty"`
+               Platforms   []Platform        `json:",omitempty"`
                Flags       []string          `json:",omitempty"`
                DriverOpts  map[string]string `json:",omitempty"`
                Files       map[string][]byte `json:",omitempty"`

Copy link
Member Author

@crazy-max crazy-max Dec 13, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you know what specifically was needed to use the platforms.Format()?

This is for the --format json so we are consistent with table / raw result.

I also tried outputing only the .Platforms, which also seems to work, although (not sure where those come from), it looks like there's empty lines in between each row 🤔

Blank lines are the builders that don't carry platforms (only nodes). I'm not sure how we can handle this with custom go template as builders and nodes are really different.

Oh, and I guess it doesn't show the default / native platform; (linux/arm64* vs linux/arm64);

* on platform means user enforces this platform when creating the builder or appending a node (create --platform linux/amd64) so it will build on this node if it matches the target platform. Atm table, raw format shows supported and preferred platforms but json format doesn't (maybe it should?).

Name string
Endpoint string
Platforms []string `json:",omitempty"`
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I know we haven't been very consistent in these, but wondering if we should include an explicit name to use in the json output; wdyt? i.e., if we explicitly want the output to be starting with an uppercase;

Suggested change
Platforms []string `json:",omitempty"`
Platforms []string `json:"Platforms,omitempty"`

Flags []string `json:",omitempty"`
DriverOpts map[string]string `json:",omitempty"`
Files map[string][]byte `json:",omitempty"`
Status string `json:",omitempty"`
ProxyConfig map[string]string `json:",omitempty"`
Version string `json:",omitempty"`
Err string `json:",omitempty"`
}{
Name: n.Name,
Endpoint: n.Endpoint,
Platforms: pp,
Flags: n.Flags,
DriverOpts: n.DriverOpts,
Files: n.Files,
Status: status,
ProxyConfig: n.ProxyConfig,
Version: n.Version,
Err: err,
})
}

func (n *Node) loadData(ctx context.Context) error {
if n.Driver == nil {
return nil
Expand Down
82 changes: 25 additions & 57 deletions commands/ls.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,23 +3,24 @@ package commands
import (
"context"
"fmt"
"io"
"strings"
"text/tabwriter"
"time"

"github.com/docker/buildx/builder"
"github.com/docker/buildx/store/storeutil"
"github.com/docker/buildx/util/cobrautil"
"github.com/docker/buildx/util/platformutil"
"github.com/docker/cli/cli"
"github.com/docker/cli/cli/command"
"github.com/docker/cli/cli/command/formatter"
"github.com/moby/buildkit/util/appcontext"
"github.com/spf13/cobra"
"golang.org/x/sync/errgroup"
)

type lsOptions struct {
quiet bool
noTrunc bool
format string
}

func runLs(dockerCli command.Cli, in lsOptions) error {
Expand All @@ -41,39 +42,26 @@ func runLs(dockerCli command.Cli, in lsOptions) error {
return err
}

timeoutCtx, cancel := context.WithTimeout(ctx, 20*time.Second)
defer cancel()

eg, _ := errgroup.WithContext(timeoutCtx)
for _, b := range builders {
func(b *builder.Builder) {
eg.Go(func() error {
_, _ = b.LoadNodes(timeoutCtx, true)
return nil
})
}(b)
}

if err := eg.Wait(); err != nil {
return err
}

w := tabwriter.NewWriter(dockerCli.Out(), 0, 0, 1, ' ', 0)
fmt.Fprintf(w, "NAME/NODE\tDRIVER/ENDPOINT\tSTATUS\tBUILDKIT\tPLATFORMS\n")

printErr := false
for _, b := range builders {
if current.Name == b.Name {
b.Name += " *"
if !in.quiet {
timeoutCtx, cancel := context.WithTimeout(ctx, 20*time.Second)
defer cancel()
eg, _ := errgroup.WithContext(timeoutCtx)
for _, b := range builders {
func(b *builder.Builder) {
eg.Go(func() error {
_, _ = b.LoadNodes(timeoutCtx, true)
return nil
})
}(b)
}
if ok := printBuilder(w, b); !ok {
printErr = true
if err := eg.Wait(); err != nil {
return err
}
}

w.Flush()

if printErr {
if hasErrors, err := lsPrint(dockerCli, current, builders, !in.noTrunc, in.quiet, in.format); err != nil {
return err
} else if hasErrors {
_, _ = fmt.Fprintf(dockerCli.Err(), "\n")
for _, b := range builders {
if b.Err() != nil {
Expand All @@ -91,31 +79,6 @@ func runLs(dockerCli command.Cli, in lsOptions) error {
return nil
}

func printBuilder(w io.Writer, b *builder.Builder) (ok bool) {
ok = true
var err string
if b.Err() != nil {
ok = false
err = "error"
}
fmt.Fprintf(w, "%s\t%s\t%s\t\t\n", b.Name, b.Driver, err)
if b.Err() == nil {
for _, n := range b.Nodes() {
var status string
if n.DriverInfo != nil {
status = n.DriverInfo.Status.String()
}
if n.Err != nil {
ok = false
fmt.Fprintf(w, " %s\t%s\t%s\t\t\n", n.Name, n.Endpoint, "error")
} else {
fmt.Fprintf(w, " %s\t%s\t%s\t%s\t%s\n", n.Name, n.Endpoint, status, n.Version, strings.Join(platformutil.FormatInGroups(n.Node.Platforms, n.Platforms), ", "))
}
}
}
return
}

func lsCmd(dockerCli command.Cli) *cobra.Command {
var options lsOptions

Expand All @@ -128,6 +91,11 @@ func lsCmd(dockerCli command.Cli) *cobra.Command {
},
}

flags := cmd.Flags()
flags.BoolVarP(&options.quiet, "quiet", "q", false, "Only display builder names")
flags.BoolVar(&options.noTrunc, "no-trunc", false, "Do not truncate output")
flags.StringVar(&options.format, "format", formatter.TableFormatKey, "Format the output")

// hide builder persistent flag for this command
cobrautil.HideInheritedFlags(cmd, "builder")

Expand Down
Loading