Improve append command - ease the process to push manifests to remo…

…te registries from a set of images

Signed-off-by: Vincent Boulineau <vincent.boulineau@datadoghq.com>

cmd/crane/cmd/append.go

@@ -18,58 +18,56 @@ import (
 	"log"
 
 	"github.com/google/go-containerregistry/pkg/crane"
-	"github.com/google/go-containerregistry/pkg/logs"
-	v1 "github.com/google/go-containerregistry/pkg/v1"
-	"github.com/google/go-containerregistry/pkg/v1/empty"
 	"github.com/spf13/cobra"
 )
 
 // NewCmdAppend creates a new cobra.Command for the append subcommand.
 func NewCmdAppend(options *[]crane.Option) *cobra.Command {
-	var baseRef, newTag, outFile string
+	var baseRef, target, outFile string
 	var newLayers []string
+	var newImages []string
 
 	appendCmd := &cobra.Command{
 		Use:   "append",
-		Short: "Append contents of a tarball to a remote image",
+		Short: "Append contents of a tarball/images to a remote image/index",
 		Args:  cobra.NoArgs,
-		Run: func(_ *cobra.Command, args []string) {
-			var base v1.Image
-			var err error
-
-			if baseRef == "" {
-				logs.Warn.Printf("base unspecified, using empty image")
-				base = empty.Image
-
-			} else {
-				base, err = crane.Pull(baseRef, *options...)
-				if err != nil {
-					log.Fatalf("pulling %s: %v", baseRef, err)
-				}
-			}
-
-			img, err := crane.Append(base, newLayers...)
+		Run: func(_ *cobra.Command, _ []string) {
+			img, idx, err := crane.Append(baseRef, newLayers, newImages, *options...)
 			if err != nil {
-				log.Fatalf("appending %v: %v", newLayers, err)
+				log.Fatalf("appending %v/%v: %v", newLayers, newImages, err)
 			}
 
 			if outFile != "" {
-				if err := crane.Save(img, newTag, outFile); err != nil {
-					log.Fatalf("writing output %q: %v", outFile, err)
+				if img != nil {
+					if err := crane.Save(img, target, outFile); err != nil {
+						log.Fatalf("writing output %q: %v", outFile, err)
+					}
+				}
+
+				if idx != nil {
+					log.Fatalf("writing file output not supported for index")
 				}
 			} else {
-				if err := crane.Push(img, newTag, *options...); err != nil {
-					log.Fatalf("pushing image %s: %v", newTag, err)
+				if img != nil {
+					if err := crane.Push(img, target, *options...); err != nil {
+						log.Fatalf("pushing image %s: %v", target, err)
+					}
+				}
+
+				if idx != nil {
+					if err := crane.PushIndex(idx, target, *options...); err != nil {
+						log.Fatalf("pushing image %s: %v", target, err)
+					}
 				}
 			}
 		},
 	}
-	appendCmd.Flags().StringVarP(&baseRef, "base", "b", "", "Name of base image to append to")
-	appendCmd.Flags().StringVarP(&newTag, "new_tag", "t", "", "Tag to apply to resulting image")
-	appendCmd.Flags().StringSliceVarP(&newLayers, "new_layer", "f", []string{}, "Path to tarball to append to image")
+	appendCmd.Flags().StringVarP(&baseRef, "base", "b", "", "Name of base image/index to append to")
+	appendCmd.Flags().StringVarP(&target, "target", "t", "", "Target name to publish image/index to")
+	appendCmd.Flags().StringSliceVarP(&newLayers, "new_layer", "f", []string{}, "Path to tarball to append to image/index")
+	appendCmd.Flags().StringSliceVarP(&newImages, "new_image", "i", []string{}, "References to remote image")
 	appendCmd.Flags().StringVarP(&outFile, "output", "o", "", "Path to new tarball of resulting image")
 
-	appendCmd.MarkFlagRequired("new_tag")
-	appendCmd.MarkFlagRequired("new_layer")
+	appendCmd.MarkFlagRequired("target")
 	return appendCmd
 }

pkg/crane/append.go

@@ -18,14 +18,136 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/empty"
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/google/go-containerregistry/pkg/v1/stream"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
+	"github.com/google/go-containerregistry/pkg/v1/types"
 )
 
 // Append reads a layer from path and appends it the the v1.Image base.
-func Append(base v1.Image, paths ...string) (v1.Image, error) {
+func Append(base string, paths []string, images []string, opt ...Option) (v1.Image, v1.ImageIndex, error) {
+	o := makeOptions(opt...)
+
+	baseImg, baseIndex, err := getBaseDescriptor(base, o)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if baseImg == nil && baseIndex == nil {
+		if len(images) > 0 {
+			baseIndex = mutate.IndexMediaType(empty.Index, types.DockerManifestList)
+		} else {
+			baseImg = empty.Image
+		}
+	}
+
+	if baseImg != nil {
+		if len(images) > 0 {
+			return nil, nil, fmt.Errorf("unable to append images to images - don't set base to produce an index instead")
+		}
+
+		img, err := appendImage(baseImg, paths)
+		return img, nil, err
+	}
+
+	if baseIndex != nil {
+		idx, err := appendIndex(baseIndex, paths, images, o)
+		return nil, idx, err
+	}
+
+	return nil, nil, fmt.Errorf("unable to determine base image")
+}
+
+func appendImage(base v1.Image, paths []string) (v1.Image, error) {
+	layers, err := getLayers(paths)
+	if err != nil {
+		return nil, err
+	}
+
+	return mutate.AppendLayers(base, layers...)
+}
+
+func appendIndex(base v1.ImageIndex, paths []string, images []string, o options) (v1.ImageIndex, error) {
+	destIndex := base
+
+	for _, path := range paths {
+		layer, err := getLayer(path)
+		if err != nil {
+			return nil, err
+		}
+		adds, err := indexAddendumFromLayer(layer)
+		if err != nil {
+			return nil, err
+		}
+
+		destIndex = mutate.AppendManifests(destIndex, adds...)
+	}
+
+	for _, image := range images {
+		desc, err := getRemoteDescriptor(image, o)
+		if err != nil {
+			return nil, err
+		}
+		adds, err := indexAddendumFromRemote(desc)
+		if err != nil {
+			return nil, err
+		}
+
+		destIndex = mutate.AppendManifests(destIndex, adds...)
+	}
+
+	return destIndex, nil
+}
+
+func getRemoteDescriptor(src string, o options) (*remote.Descriptor, error) {
+	ref, err := name.ParseReference(src, o.name...)
+	if err != nil {
+		return nil, fmt.Errorf("parsing reference for %q: %v", src, err)
+	}
+
+	desc, err := remote.Get(ref, o.remote...)
+	if err != nil {
+		return nil, fmt.Errorf("fetching %q: %v", src, err)
+	}
+
+	return desc, nil
+}
+
+func getBaseDescriptor(base string, o options) (v1.Image, v1.ImageIndex, error) {
+	if base == "" {
+		return nil, nil, nil
+	}
+
+	desc, err := getRemoteDescriptor(base, o)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	switch desc.MediaType {
+	case types.OCIImageIndex, types.DockerManifestList:
+		idx, err := desc.ImageIndex()
+		if err != nil {
+			return nil, nil, err
+		}
+		return nil, idx, err
+
+	case types.DockerManifestSchema1, types.DockerManifestSchema1Signed:
+		return nil, nil, fmt.Errorf("append to a v1 manifest is not supported")
+
+	default:
+		img, err := desc.Image()
+		if err != nil {
+			return nil, nil, err
+		}
+		return img, nil, err
+	}
+}
+
+func getLayers(paths []string) ([]v1.Layer, error) {
 	layers := make([]v1.Layer, 0, len(paths))
 	for _, path := range paths {
 		layer, err := getLayer(path)
@@ -36,7 +158,7 @@ func Append(base v1.Image, paths ...string) (v1.Image, error) {
 		layers = append(layers, layer)
 	}
 
-	return mutate.AppendLayers(base, layers...)
+	return layers, nil
 }
 
 func getLayer(path string) (v1.Layer, error) {
@@ -57,3 +179,94 @@ func getLayer(path string) (v1.Layer, error) {
 
 	return tarball.LayerFromFile(path)
 }
+
+func indexAddendumFromLayer(layer v1.Layer) ([]mutate.IndexAddendum, error) {
+	adds := make([]mutate.IndexAddendum, 0)
+
+	mediaType, err := layer.MediaType()
+	if err != nil {
+		return nil, err
+	}
+
+	size, err := layer.Size()
+	if err != nil {
+		return nil, err
+	}
+
+	hash, err := layer.Digest()
+	if err != nil {
+		return nil, err
+	}
+
+	adds = append(adds, mutate.IndexAddendum{
+		Add: layer,
+		Descriptor: v1.Descriptor{
+			MediaType: mediaType,
+			Size:      size,
+			Digest:    hash,
+		},
+	})
+
+	return adds, nil
+}
+
+func indexAddendumFromRemote(desc *remote.Descriptor) ([]mutate.IndexAddendum, error) {
+	adds := make([]mutate.IndexAddendum, 0)
+
+	switch desc.MediaType {
+	case types.OCIImageIndex, types.DockerManifestList:
+		idx, err := desc.ImageIndex()
+		if err != nil {
+			return nil, err
+		}
+
+		im, err := idx.IndexManifest()
+		if err != nil {
+			return nil, err
+		}
+
+		for _, imDesc := range im.Manifests {
+			img, err := idx.Image(imDesc.Digest)
+			if err != nil {
+				return nil, err
+			}
+
+			adds = append(adds, mutate.IndexAddendum{
+				Add: img,
+				Descriptor: v1.Descriptor{
+					URLs:        imDesc.URLs,
+					MediaType:   imDesc.MediaType,
+					Annotations: imDesc.Annotations,
+					Platform:    imDesc.Platform,
+				},
+			})
+		}
+	case types.DockerManifestSchema1, types.DockerManifestSchema1Signed:
+		return nil, fmt.Errorf("merging v1 manifest is not supported")
+	default:
+		img, err := desc.Image()
+		if err != nil {
+			return nil, err
+		}
+		cfg, err := img.ConfigFile()
+		if err != nil {
+			return nil, err
+		}
+
+		adds = append(adds, mutate.IndexAddendum{
+			Add: img,
+			Descriptor: v1.Descriptor{
+				URLs:        desc.URLs,
+				MediaType:   desc.MediaType,
+				Annotations: desc.Annotations,
+				Platform: &v1.Platform{
+					Architecture: cfg.Architecture,
+					OS:           cfg.OS,
+					OSVersion:    cfg.OSVersion,
+				},
+			},
+		})
+	}
+
+	return adds, nil
+}