Use Data field when fetching in remote (#1076)

* Add Data to descriptor

* Use Data field when fetching in remote

Small optimization to use the Data field, if present.

* Update mutate package Data field interactions

* ./hack/update-codegen.sh

* Actually verify things

* verify test

internal/verify/verify.go

@@ -17,7 +17,9 @@
 package verify
 
 import (
+	"bytes"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"hash"
 	"io"
@@ -81,3 +83,25 @@ func ReadCloser(r io.ReadCloser, size int64, h v1.Hash) (io.ReadCloser, error) {
 		CloseFunc: r.Close,
 	}, nil
 }
+
+// Descriptor verifies that the embedded Data field matches the Size and Digest
+// fields of the given v1.Descriptor, returning an error if the Data field is
+// missing or if it contains incorrect data.
+func Descriptor(d v1.Descriptor) error {
+	if d.Data == nil {
+		return errors.New("error verifying descriptor; Data == nil")
+	}
+
+	h, sz, err := v1.SHA256(bytes.NewReader(d.Data))
+	if err != nil {
+		return err
+	}
+	if h != d.Digest {
+		return fmt.Errorf("error verifying Digest; got %q, want %q", h, d.Digest)
+	}
+	if sz != d.Size {
+		return fmt.Errorf("error verifying Size; got %d, want %d", sz, d.Size)
+	}
+
+	return nil
+}

internal/verify/verify_test.go

@@ -16,6 +16,7 @@ package verify
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"strings"
@@ -86,3 +87,47 @@ func TestBadSize(t *testing.T) {
 		})
 	}
 }
+
+func TestDescriptor(t *testing.T) {
+	for _, tc := range []struct {
+		err  error
+		desc v1.Descriptor
+	}{{
+		err: errors.New("error verifying descriptor; Data == nil"),
+	}, {
+		err: errors.New(`error verifying Digest; got "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", want ":"`),
+		desc: v1.Descriptor{
+			Data: []byte("abc"),
+		},
+	}, {
+		err: errors.New("error verifying Size; got 3, want 0"),
+		desc: v1.Descriptor{
+			Data: []byte("abc"),
+			Digest: v1.Hash{
+				Algorithm: "sha256",
+				Hex:       "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
+			},
+		},
+	}, {
+		desc: v1.Descriptor{
+			Data: []byte("abc"),
+			Size: 3,
+			Digest: v1.Hash{
+				Algorithm: "sha256",
+				Hex:       "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
+			},
+		},
+	}} {
+		got, want := Descriptor(tc.desc), tc.err
+
+		if got == nil {
+			if want != nil {
+				t.Errorf("Descriptor(): got nil, want %v", want)
+			}
+		} else if want == nil {
+			t.Errorf("Descriptor(): got %v, want nil", got)
+		} else if got, want := got.Error(), want.Error(); got != want {
+			t.Errorf("Descriptor(): got %q, want %q", got, want)
+		}
+	}
+}

pkg/v1/manifest.go

@@ -43,6 +43,7 @@ type Descriptor struct {
 	MediaType   types.MediaType   `json:"mediaType"`
 	Size        int64             `json:"size"`
 	Digest      Hash              `json:"digest"`
+	Data        []byte            `json:"data,omitempty"`
 	URLs        []string          `json:"urls,omitempty"`
 	Annotations map[string]string `json:"annotations,omitempty"`
 	Platform    *Platform         `json:"platform,omitempty"`

pkg/v1/mutate/image.go

@@ -130,6 +130,11 @@ func (i *image) compute() error {
 	manifest.Config.Digest = d
 	manifest.Config.Size = sz
 
+	// If Data was set in the base image, we need to update it in the mutated image.
+	if m.Config.Data != nil {
+		manifest.Config.Data = rcfg
+	}
+
 	// With OCI media types, this should not be set, see discussion:
 	// https://github.com/opencontainers/image-spec/pull/795
 	if i.mediaType != nil {

pkg/v1/mutate/index.go

@@ -51,6 +51,9 @@ func computeDescriptor(ia IndexAddendum) (*v1.Descriptor, error) {
 	if len(ia.Descriptor.Annotations) != 0 {
 		desc.Annotations = ia.Descriptor.Annotations
 	}
+	if ia.Descriptor.Data != nil {
+		desc.Data = ia.Descriptor.Data
+	}
 
 	return desc, nil
 }

pkg/v1/remote/image.go

@@ -15,6 +15,7 @@
 package remote
 
 import (
+	"bytes"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -100,6 +101,14 @@ func (r *remoteImage) RawConfigFile() ([]byte, error) {
 		return nil, err
 	}
 
+	if m.Config.Data != nil {
+		if err := verify.Descriptor(m.Config); err != nil {
+			return nil, err
+		}
+		r.config = m.Config.Data
+		return r.config, nil
+	}
+
 	body, err := r.fetchBlob(r.context, m.Config.Size, m.Config.Digest)
 	if err != nil {
 		return nil, err
@@ -143,6 +152,10 @@ func (rl *remoteImageLayer) Compressed() (io.ReadCloser, error) {
 		return nil, err
 	}
 
+	if d.Data != nil {
+		return verify.ReadCloser(ioutil.NopCloser(bytes.NewReader(d.Data)), d.Size, d.Digest)
+	}
+
 	// We don't want to log binary layers -- this can break terminals.
 	ctx := redact.NewContext(rl.ri.context, "omitting binary blobs from logs")
 

pkg/v1/remote/image_test.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -403,6 +404,8 @@ func TestPullingManifestList(t *testing.T) {
 	manifest.Manifests[0].Platform = &fakePlatform
 	// Make sure the second manifest does.
 	manifest.Manifests[1].Platform = &defaultPlatform
+	// Do short-circuiting via Data.
+	manifest.Manifests[1].Data = mustRawManifest(t, child)
 	rawManifest, err := json.Marshal(manifest)
 	if err != nil {
 		t.Fatal(err)
@@ -674,3 +677,65 @@ func TestPullingForeignLayer(t *testing.T) {
 		t.Errorf("failed to Write: %v", err)
 	}
 }
+
+func TestData(t *testing.T) {
+	img := randomImage(t)
+	manifest, err := img.Manifest()
+	if err != nil {
+		t.Fatal(err)
+	}
+	layers, err := img.Layers()
+	if err != nil {
+		t.Fatal(err)
+	}
+	cb, err := img.RawConfigFile()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	manifest.Config.Data = cb
+	rc, err := layers[0].Compressed()
+	if err != nil {
+		t.Fatal(err)
+	}
+	lb, err := ioutil.ReadAll(rc)
+	if err != nil {
+		t.Fatal(err)
+	}
+	manifest.Layers[0].Data = lb
+	rawManifest, err := json.Marshal(manifest)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/v2/":
+			w.WriteHeader(http.StatusOK)
+		case "/v2/test/manifests/latest":
+			if r.Method != http.MethodGet {
+				t.Errorf("Method; got %v, want %v", r.Method, http.MethodGet)
+			}
+			w.Write(rawManifest)
+		default:
+			// explode if we try to read blob or config
+			t.Fatalf("Unexpected path: %v", r.URL.Path)
+		}
+	}))
+	defer server.Close()
+	u, err := url.Parse(server.URL)
+	if err != nil {
+		t.Fatalf("url.Parse(%v) = %v", server.URL, err)
+	}
+	ref, err := newReference(u.Host, "test", "latest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	rmt, err := Image(ref)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := validate.Image(rmt); err != nil {
+		t.Fatal(err)
+	}
+}

pkg/v1/remote/index.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/google/go-containerregistry/internal/verify"
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/partial"
@@ -198,9 +199,20 @@ func (r *remoteIndex) childByHash(h v1.Hash) (*Descriptor, error) {
 // Convert one of this index's child's v1.Descriptor into a remote.Descriptor, with the given platform option.
 func (r *remoteIndex) childDescriptor(child v1.Descriptor, platform v1.Platform) (*Descriptor, error) {
 	ref := r.Ref.Context().Digest(child.Digest.String())
-	manifest, _, err := r.fetchManifest(ref, []types.MediaType{child.MediaType})
-	if err != nil {
-		return nil, err
+	var (
+		manifest []byte
+		err      error
+	)
+	if child.Data != nil {
+		if err := verify.Descriptor(child); err != nil {
+			return nil, err
+		}
+		manifest = child.Data
+	} else {
+		manifest, _, err = r.fetchManifest(ref, []types.MediaType{child.MediaType})
+		if err != nil {
+			return nil, err
+		}
 	}
 	return &Descriptor{
 		fetcher: fetcher{