Add remote.Puller

This PR adds a Puller implementation and uses it in `crane ls` and
`crane catalog` to stream results for each page.

cmd/crane/cmd/catalog.go

@@ -15,40 +15,62 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
-	"os"
-	"strings"
+	"path"
 
 	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/spf13/cobra"
 )
 
-// NewCmdCatalog creates a new cobra.Command for the repos subcommand.
+// NewCmdCatalog creates a new cobra.Command for the catalog subcommand.
 func NewCmdCatalog(options *[]crane.Option, argv ...string) *cobra.Command {
-	if len(argv) == 0 {
-		argv = []string{os.Args[0]}
+	var fullRef bool
+	cmd := &cobra.Command{
+		Use:   "catalog REGISTRY",
+		Short: "List the repos in a registry",
+		Args:  cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			o := crane.GetOptions(*options...)
+
+			return catalog(cmd.Context(), args[0], fullRef, o)
+		},
 	}
+	cmd.Flags().BoolVar(&fullRef, "full-ref", false, "(Optional) if true, print the full image reference")
 
-	baseCmd := strings.Join(argv, " ")
-	eg := fmt.Sprintf(`  # list the repos for reg.example.com
-  $ %s catalog reg.example.com`, baseCmd)
-
-	return &cobra.Command{
-		Use:     "catalog [REGISTRY]",
-		Short:   "List the repos in a registry",
-		Example: eg,
-		Args:    cobra.ExactArgs(1),
-		RunE: func(_ *cobra.Command, args []string) error {
-			reg := args[0]
-			repos, err := crane.Catalog(reg, *options...)
-			if err != nil {
-				return fmt.Errorf("reading repos for %s: %w", reg, err)
-			}
+	return cmd
+}
+
+func catalog(ctx context.Context, src string, fullRef bool, o crane.Options) error {
+	reg, err := name.NewRegistry(src, o.Name...)
+	if err != nil {
+		return fmt.Errorf("parsing reg %q: %w", src, err)
+	}
 
-			for _, repo := range repos {
+	puller, err := remote.NewPuller(o.Remote...)
+	if err != nil {
+		return err
+	}
+
+	catalogger, err := puller.Catalogger(ctx, reg)
+	if err != nil {
+		return fmt.Errorf("reading tags for %s: %w", reg, err)
+	}
+
+	for catalogger.HasNext() {
+		repos, err := catalogger.Next(ctx)
+		if err != nil {
+			return err
+		}
+		for _, repo := range repos.Repos {
+			if fullRef {
+				fmt.Println(path.Join(src, repo))
+			} else {
 				fmt.Println(repo)
 			}
-			return nil
-		},
+		}
 	}
+	return nil
 }

cmd/crane/cmd/list.go

@@ -15,11 +15,13 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"strings"
 
 	"github.com/google/go-containerregistry/pkg/crane"
 	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/spf13/cobra"
 )
 
@@ -30,33 +32,49 @@ func NewCmdList(options *[]crane.Option) *cobra.Command {
 		Use:   "ls REPO",
 		Short: "List the tags in a repo",
 		Args:  cobra.ExactArgs(1),
-		RunE: func(_ *cobra.Command, args []string) error {
-			repo := args[0]
-			tags, err := crane.ListTags(repo, *options...)
-			if err != nil {
-				return fmt.Errorf("reading tags for %s: %w", repo, err)
-			}
-
-			r, err := name.NewRepository(repo)
-			if err != nil {
-				return err
-			}
+		RunE: func(cmd *cobra.Command, args []string) error {
+			o := crane.GetOptions(*options...)
 
-			for _, tag := range tags {
-				if omitDigestTags && strings.HasPrefix(tag, "sha256-") {
-					continue
-				}
-
-				if fullRef {
-					fmt.Println(r.Tag(tag))
-				} else {
-					fmt.Println(tag)
-				}
-			}
-			return nil
+			return list(cmd.Context(), args[0], fullRef, omitDigestTags, o)
 		},
 	}
 	cmd.Flags().BoolVar(&fullRef, "full-ref", false, "(Optional) if true, print the full image reference")
 	cmd.Flags().BoolVar(&omitDigestTags, "omit-digest-tags", false, "(Optional), if true, omit digest tags (e.g., ':sha256-...')")
 	return cmd
 }
+
+func list(ctx context.Context, src string, fullRef, omitDigestTags bool, o crane.Options) error {
+	repo, err := name.NewRepository(src, o.Name...)
+	if err != nil {
+		return fmt.Errorf("parsing repo %q: %w", src, err)
+	}
+
+	puller, err := remote.NewPuller(o.Remote...)
+	if err != nil {
+		return err
+	}
+
+	lister, err := puller.Lister(ctx, repo)
+	if err != nil {
+		return fmt.Errorf("reading tags for %s: %w", repo, err)
+	}
+
+	for lister.HasNext() {
+		tags, err := lister.Next(ctx)
+		if err != nil {
+			return err
+		}
+		for _, tag := range tags.Tags {
+			if omitDigestTags && strings.HasPrefix(tag, "sha256-") {
+				continue
+			}
+
+			if fullRef {
+				fmt.Println(repo.Tag(tag))
+			} else {
+				fmt.Println(tag)
+			}
+		}
+	}
+	return nil
+}

pkg/v1/remote/catalog.go

@@ -25,8 +25,9 @@ import (
 	"github.com/google/go-containerregistry/pkg/v1/remote/transport"
 )
 
-type catalog struct {
+type Catalogs struct {
 	Repos []string `json:"repositories"`
+	Next  string   `json:"next,omitempty"`
 }
 
 // CatalogPage calls /_catalog, returning the list of repositories on the registry.
@@ -61,7 +62,7 @@ func CatalogPage(target name.Registry, last string, n int, options ...Option) ([
 		return nil, err
 	}
 
-	var parsed catalog
+	var parsed Catalogs
 	if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
 		return nil, err
 	}
@@ -75,70 +76,82 @@ func Catalog(ctx context.Context, target name.Registry, options ...Option) ([]st
 	if err != nil {
 		return nil, err
 	}
-	f, err := makeFetcher(o.context, target, o)
+
+	// WithContext overrides the ctx passed directly.
+	if o.context != context.Background() {
+		ctx = o.context
+	}
+
+	return newPuller(o).Catalog(ctx, target)
+}
+
+func (f *fetcher) catalogPage(ctx context.Context, reg name.Registry, next string) (*Catalogs, error) {
+	if next == "" {
+		uri := &url.URL{
+			Scheme: reg.Scheme(),
+			Host:   reg.RegistryStr(),
+			Path:   "/v2/_catalog",
+		}
+		if f.pageSize > 0 {
+			uri.RawQuery = fmt.Sprintf("n=%d", f.pageSize)
+		}
+		next = uri.String()
+	}
+
+	req, err := http.NewRequestWithContext(ctx, "GET", next, nil)
 	if err != nil {
 		return nil, err
 	}
 
-	uri := &url.URL{
-		Scheme: target.Scheme(),
-		Host:   target.RegistryStr(),
-		Path:   "/v2/_catalog",
+	resp, err := f.client.Do(req)
+	if err != nil {
+		return nil, err
 	}
-	if o.pageSize > 0 {
-		uri.RawQuery = fmt.Sprintf("n=%d", o.pageSize)
+
+	if err := transport.CheckError(resp, http.StatusOK); err != nil {
+		return nil, err
 	}
 
-	// WithContext overrides the ctx passed directly.
-	if o.context != context.Background() {
-		ctx = o.context
+	parsed := Catalogs{}
+	if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
+		return nil, err
 	}
 
-	var (
-		parsed   catalog
-		repoList []string
-	)
+	if err := resp.Body.Close(); err != nil {
+		return nil, err
+	}
 
-	// get responses until there is no next page
-	for {
-		select {
-		case <-ctx.Done():
-			return nil, ctx.Err()
-		default:
-		}
+	uri, err := getNextPageURL(resp)
+	if err != nil {
+		return nil, err
+	}
 
-		req, err := http.NewRequest("GET", uri.String(), nil)
-		if err != nil {
-			return nil, err
-		}
-		req = req.WithContext(ctx)
+	if uri != nil {
+		parsed.Next = uri.String()
+	}
 
-		resp, err := f.client.Do(req)
-		if err != nil {
-			return nil, err
-		}
+	return &parsed, nil
+}
 
-		if err := transport.CheckError(resp, http.StatusOK); err != nil {
-			return nil, err
-		}
+type Catalogger struct {
+	f   *fetcher
+	reg name.Registry
 
-		if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
-			return nil, err
-		}
-		if err := resp.Body.Close(); err != nil {
-			return nil, err
-		}
+	page *Catalogs
+	err  error
 
-		repoList = append(repoList, parsed.Repos...)
+	needMore bool
+}
 
-		uri, err = getNextPageURL(resp)
-		if err != nil {
-			return nil, err
-		}
-		// no next page
-		if uri == nil {
-			break
-		}
+func (l *Catalogger) Next(ctx context.Context) (*Catalogs, error) {
+	if l.needMore {
+		l.page, l.err = l.f.catalogPage(ctx, l.reg, l.page.Next)
+	} else {
+		l.needMore = true
 	}
-	return repoList, nil
+	return l.page, l.err
+}
+
+func (l *Catalogger) HasNext() bool {
+	return l.page != nil && (!l.needMore || l.page.Next != "")
 }

pkg/v1/remote/descriptor.go

@@ -249,6 +249,7 @@ type fetcher struct {
 	client   *http.Client
 	context  context.Context
 	platform v1.Platform
+	pageSize int
 }
 
 func makeFetcher(ctx context.Context, target resource, o *options) (*fetcher, error) {
@@ -279,6 +280,7 @@ func makeFetcher(ctx context.Context, target resource, o *options) (*fetcher, er
 		client:   &http.Client{Transport: tr},
 		context:  ctx,
 		platform: o.platform,
+		pageSize: o.pageSize,
 	}, nil
 }