diff --git a/go.mod b/go.mod index dcb3c9becf..2862a80168 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960 github.com/fsnotify/fsnotify v1.4.7 github.com/google/go-cmp v0.3.0 - github.com/google/go-containerregistry v0.0.0-20200115190719-e8e9aa676278 + github.com/google/go-containerregistry v0.0.0-20200212224832-c629a66d7231 github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc // indirect github.com/mattmoor/dep-notify v0.0.0-20190205035814-a45dec370a17 github.com/spf13/cobra v0.0.5 diff --git a/go.sum b/go.sum index fa7747de79..ecf20e10d6 100644 --- a/go.sum +++ b/go.sum @@ -136,6 +136,8 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-containerregistry v0.0.0-20200115190719-e8e9aa676278 h1:jzLpEQ2Is2AOHtwZvILBecSxSxq9v3KpZU6fIQIvYYg= github.com/google/go-containerregistry v0.0.0-20200115190719-e8e9aa676278/go.mod h1:Wtl/v6YdQxv397EREtzwgd9+Ud7Q5D8XMbi3Zazgkrs= +github.com/google/go-containerregistry v0.0.0-20200212224832-c629a66d7231 h1:zoj6E1dzY9aeZw1CGJv1hffxgyunrLpjI0SZWK7ynzg= +github.com/google/go-containerregistry v0.0.0-20200212224832-c629a66d7231/go.mod h1:Wtl/v6YdQxv397EREtzwgd9+Ud7Q5D8XMbi3Zazgkrs= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/image.go index 382d139ede..e13ef1f8b7 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/image.go @@ -120,8 +120,12 @@ func (i *image) compute() error { // With OCI media types, this should not be set, see discussion: // https://github.com/opencontainers/image-spec/pull/795 - if i.mediaType != nil && strings.Contains(string(*i.mediaType), types.OCIVendorPrefix) { - manifest.MediaType = "" + if i.mediaType != nil { + if strings.Contains(string(*i.mediaType), types.OCIVendorPrefix) { + manifest.MediaType = "" + } else if strings.Contains(string(*i.mediaType), types.DockerVendorPrefix) { + manifest.MediaType = *i.mediaType + } } i.configFile = configFile diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/index.go index 21ccf50c26..b8f88c3b83 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/index.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/mutate/index.go @@ -24,31 +24,33 @@ import ( "github.com/google/go-containerregistry/pkg/v1/types" ) -func computeDescriptor(desc v1.Descriptor, add Appendable) (*v1.Descriptor, error) { - d, err := add.Digest() +func computeDescriptor(ia IndexAddendum) (*v1.Descriptor, error) { + desc, err := partial.Descriptor(ia.Add) if err != nil { return nil, err } - mt, err := add.MediaType() - if err != nil { - return nil, err + + // The IndexAddendum allows overriding Descriptor values. + if ia.Descriptor.Size != 0 { + desc.Size = ia.Descriptor.Size } - sz, err := add.Size() - if err != nil { - return nil, err + if string(ia.Descriptor.MediaType) != "" { + desc.MediaType = ia.Descriptor.MediaType } - - // The IndexAddendum allows overriding These values. - if desc.Size == 0 { - desc.Size = sz + if ia.Descriptor.Digest != (v1.Hash{}) { + desc.Digest = ia.Descriptor.Digest + } + if ia.Descriptor.Platform != nil { + desc.Platform = ia.Descriptor.Platform } - if string(desc.MediaType) == "" { - desc.MediaType = mt + if len(ia.Descriptor.URLs) != 0 { + desc.URLs = ia.Descriptor.URLs } - if desc.Digest == (v1.Hash{}) { - desc.Digest = d + if len(ia.Descriptor.Annotations) != 0 { + desc.Annotations = ia.Descriptor.Annotations } - return &desc, nil + + return desc, nil } type index struct { @@ -89,7 +91,7 @@ func (i *index) compute() error { manifest := m.DeepCopy() manifests := manifest.Manifests for _, add := range i.adds { - desc, err := computeDescriptor(add.Descriptor, add.Add) + desc, err := computeDescriptor(add) if err != nil { return err } @@ -107,8 +109,12 @@ func (i *index) compute() error { // With OCI media types, this should not be set, see discussion: // https://github.com/opencontainers/image-spec/pull/795 - if i.mediaType != nil && strings.Contains(string(*i.mediaType), types.OCIVendorPrefix) { - manifest.MediaType = "" + if i.mediaType != nil { + if strings.Contains(string(*i.mediaType), types.OCIVendorPrefix) { + manifest.MediaType = "" + } else if strings.Contains(string(*i.mediaType), types.DockerVendorPrefix) { + manifest.MediaType = *i.mediaType + } } i.manifest = manifest diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go index c75d17a0e5..08bdce3e07 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/partial/with.go @@ -129,12 +129,6 @@ func RawConfigFile(i WithConfigFile) ([]byte, error) { return json.Marshal(cfg) } -// WithUncompressedLayer defines the subset of v1.Image used by these helper methods -type WithUncompressedLayer interface { - // UncompressedLayer is like UncompressedBlob, but takes the "diff id". - UncompressedLayer(v1.Hash) (io.ReadCloser, error) -} - // WithRawManifest defines the subset of v1.Image used by these helper methods type WithRawManifest interface { // RawManifest returns the serialized bytes of this image's config file. @@ -348,3 +342,41 @@ func Descriptor(d Describable) (*v1.Descriptor, error) { return &desc, nil } + +type withUncompressedSize interface { + UncompressedSize() (int64, error) +} + +// UncompressedSize returns the size of the Uncompressed layer. If the +// underlying implementation doesn't implement UncompressedSize directly, +// this will compute the uncompressedSize by reading everything returned +// by Compressed(). This is potentially expensive and may consume the contents +// for streaming layers. +func UncompressedSize(l v1.Layer) (int64, error) { + // If the layer implements UncompressedSize itself, return that. + if wus, ok := l.(withUncompressedSize); ok { + return wus.UncompressedSize() + } + + // Otherwise, try to unwrap any partial implementations to see + // if the wrapped struct implements UncompressedSize. + if ule, ok := l.(*uncompressedLayerExtender); ok { + if wus, ok := ule.UncompressedLayer.(withUncompressedSize); ok { + return wus.UncompressedSize() + } + } + if cle, ok := l.(*compressedLayerExtender); ok { + if wus, ok := cle.CompressedLayer.(withUncompressedSize); ok { + return wus.UncompressedSize() + } + } + + // The layer doesn't implement UncompressedSize, we need to compute it. + rc, err := l.Uncompressed() + if err != nil { + return -1, err + } + defer rc.Close() + + return io.Copy(ioutil.Discard, rc) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go index da0fa24c39..65ee4e7f7e 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/check.go @@ -34,7 +34,7 @@ func CheckPushPermission(ref name.Reference, kc authn.Keychain, t http.RoundTrip // authorize a push. Figure out how to return early here when we can, // to avoid a roundtrip for spec-compliant registries. w := writer{ - ref: ref, + repo: ref.Context(), client: &http.Client{Transport: tr}, } loc, _, err := w.initiateUpload("", "") diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go index 35e5d798a3..a215e794ff 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/error.go @@ -19,9 +19,27 @@ import ( "fmt" "io/ioutil" "net/http" + "net/url" "strings" ) +// The set of query string keys that we expect to send as part of the registry +// protocol. Anything else is potentially dangerous to leak, as it's probably +// from a redirect. These redirects often included tokens or signed URLs. +var paramWhitelist = map[string]struct{}{ + // Token exchange + "scope": struct{}{}, + "service": struct{}{}, + // Cross-repo mounting + "mount": struct{}{}, + "from": struct{}{}, + // Layer PUT + "digest": struct{}{}, + // Listing tags and catalog + "n": struct{}{}, + "last": struct{}{}, +} + // Error implements error to support the following error specification: // https://github.com/docker/distribution/blob/master/docs/spec/api.md#errors type Error struct { @@ -30,6 +48,8 @@ type Error struct { StatusCode int // The raw body if we couldn't understand it. rawBody string + // The request that failed. + request *http.Request } // Check that Error implements error @@ -37,6 +57,14 @@ var _ error = (*Error)(nil) // Error implements error func (e *Error) Error() string { + prefix := "" + if e.request != nil { + prefix = fmt.Sprintf("%s %s: ", e.request.Method, redact(e.request.URL)) + } + return prefix + e.responseErr() +} + +func (e *Error) responseErr() string { switch len(e.Errors) { case 0: if len(e.rawBody) == 0 { @@ -51,7 +79,7 @@ func (e *Error) Error() string { errors = append(errors, d.String()) } return fmt.Sprintf("multiple errors returned: %s", - strings.Join(errors, ";")) + strings.Join(errors, "; ")) } } @@ -69,6 +97,21 @@ func (e *Error) Temporary() bool { return true } +func redact(original *url.URL) *url.URL { + qs := original.Query() + for k, v := range qs { + for i := range v { + if _, ok := paramWhitelist[k]; !ok { + // key is not in the whitelist + v[i] = "REDACTED" + } + } + } + redacted := *original + redacted.RawQuery = qs.Encode() + return &redacted +} + // Diagnostic represents a single error returned by a Docker registry interaction. type Diagnostic struct { Code ErrorCode `json:"code"` @@ -127,5 +170,6 @@ func CheckError(resp *http.Response, codes ...int) error { structuredError.rawBody = string(b) } structuredError.StatusCode = resp.StatusCode + structuredError.request = resp.Request return structuredError } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go index 7be851f088..7b431c077c 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/remote/write.go @@ -50,13 +50,13 @@ func Write(ref name.Reference, img v1.Image, options ...Option) error { return err } - scopes := scopesForUploadingImage(ref, ls) + scopes := scopesForUploadingImage(ref.Context(), ls) tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, scopes) if err != nil { return err } w := writer{ - ref: ref, + repo: ref.Context(), client: &http.Client{Transport: tr}, } @@ -125,20 +125,20 @@ func Write(ref name.Reference, img v1.Image, options ...Option) error { // With all of the constituent elements uploaded, upload the manifest // to commit the image. - return w.commitImage(img) + return w.commitImage(img, ref) } // writer writes the elements of an image to a remote image reference. type writer struct { - ref name.Reference + repo name.Repository client *http.Client } // url returns a url.Url for the specified path in the context of this remote image reference. func (w *writer) url(path string) url.URL { return url.URL{ - Scheme: w.ref.Context().Registry.Scheme(), - Host: w.ref.Context().RegistryStr(), + Scheme: w.repo.Registry.Scheme(), + Host: w.repo.RegistryStr(), Path: path, } } @@ -164,7 +164,7 @@ func (w *writer) nextLocation(resp *http.Response) (string, error) { // initiation if "mount" is specified, even if no "from" sources are specified. // However, this is not broadly applicable to all registries, e.g. ECR. func (w *writer) checkExistingBlob(h v1.Hash) (bool, error) { - u := w.url(fmt.Sprintf("/v2/%s/blobs/%s", w.ref.Context().RepositoryStr(), h.String())) + u := w.url(fmt.Sprintf("/v2/%s/blobs/%s", w.repo.RepositoryStr(), h.String())) resp, err := w.client.Head(u.String()) if err != nil { @@ -182,7 +182,7 @@ func (w *writer) checkExistingBlob(h v1.Hash) (bool, error) { // checkExistingManifest checks if a manifest exists already in the repository // by making a HEAD request to the manifest API. func (w *writer) checkExistingManifest(h v1.Hash, mt types.MediaType) (bool, error) { - u := w.url(fmt.Sprintf("/v2/%s/manifests/%s", w.ref.Context().RepositoryStr(), h.String())) + u := w.url(fmt.Sprintf("/v2/%s/manifests/%s", w.repo.RepositoryStr(), h.String())) req, err := http.NewRequest(http.MethodHead, u.String(), nil) if err != nil { @@ -210,7 +210,7 @@ func (w *writer) checkExistingManifest(h v1.Hash, mt types.MediaType) (bool, err // upload was initiated and the body of that blob should be sent to the returned // location. func (w *writer) initiateUpload(from, mount string) (location string, mounted bool, err error) { - u := w.url(fmt.Sprintf("/v2/%s/blobs/uploads/", w.ref.Context().RepositoryStr())) + u := w.url(fmt.Sprintf("/v2/%s/blobs/uploads/", w.repo.RepositoryStr())) uv := url.Values{} if mount != "" && from != "" { // Quay will fail if we specify a "mount" without a "from". @@ -313,7 +313,7 @@ func (w *writer) uploadOne(l v1.Layer) error { mount = h.String() } if ml, ok := l.(*MountableLayer); ok { - if w.ref.Context().RegistryStr() == ml.Reference.Context().RegistryStr() { + if w.repo.RegistryStr() == ml.Reference.Context().RegistryStr() { from = ml.Reference.Context().RepositoryStr() } } @@ -407,7 +407,7 @@ func unpackTaggable(t Taggable) (*v1.Descriptor, error) { } // commitImage does a PUT of the image's manifest. -func (w *writer) commitImage(t Taggable) error { +func (w *writer) commitImage(t Taggable, ref name.Reference) error { raw, err := t.RawManifest() if err != nil { return err @@ -417,7 +417,7 @@ func (w *writer) commitImage(t Taggable) error { return err } - u := w.url(fmt.Sprintf("/v2/%s/manifests/%s", w.ref.Context().RepositoryStr(), w.ref.Identifier())) + u := w.url(fmt.Sprintf("/v2/%s/manifests/%s", w.repo.RepositoryStr(), ref.Identifier())) // Make the request to PUT the serialized manifest req, err := http.NewRequest(http.MethodPut, u.String(), bytes.NewBuffer(raw)) @@ -437,11 +437,11 @@ func (w *writer) commitImage(t Taggable) error { } // The image was successfully pushed! - logs.Progress.Printf("%v: digest: %v size: %d", w.ref, desc.Digest, len(raw)) + logs.Progress.Printf("%v: digest: %v size: %d", ref, desc.Digest, len(raw)) return nil } -func scopesForUploadingImage(ref name.Reference, layers []v1.Layer) []string { +func scopesForUploadingImage(repo name.Repository, layers []v1.Layer) []string { // use a map as set to remove duplicates scope strings scopeSet := map[string]struct{}{} @@ -449,7 +449,7 @@ func scopesForUploadingImage(ref name.Reference, layers []v1.Layer) []string { if ml, ok := l.(*MountableLayer); ok { // we will add push scope for ref.Context() after the loop. // for now we ask pull scope for references of the same registry - if ml.Reference.Context() != ref.Context() && ml.Reference.Context().Registry == ref.Context().Registry { + if ml.Reference.Context() != repo && ml.Reference.Context().Registry == repo.Registry { scopeSet[ml.Reference.Scope(transport.PullScope)] = struct{}{} } } @@ -457,7 +457,7 @@ func scopesForUploadingImage(ref name.Reference, layers []v1.Layer) []string { scopes := make([]string, 0) // Push scope should be the first element because a few registries just look at the first scope to determine access. - scopes = append(scopes, ref.Scope(transport.PushScope)) + scopes = append(scopes, repo.Scope(transport.PushScope)) for scope := range scopeSet { scopes = append(scopes, scope) @@ -485,7 +485,7 @@ func WriteIndex(ref name.Reference, ii v1.ImageIndex, options ...Option) error { return err } w := writer{ - ref: ref, + repo: ref.Context(), client: &http.Client{Transport: tr}, } @@ -523,22 +523,22 @@ func WriteIndex(ref name.Reference, ii v1.ImageIndex, options ...Option) error { // With all of the constituent elements uploaded, upload the manifest // to commit the image. - return w.commitImage(ii) + return w.commitImage(ii, ref) } -// WriteLayer uploads the provided Layer to the specified name.Digest. -func WriteLayer(ref name.Digest, layer v1.Layer, options ...Option) error { - o, err := makeOptions(ref.Context(), options...) +// WriteLayer uploads the provided Layer to the specified repo. +func WriteLayer(repo name.Repository, layer v1.Layer, options ...Option) error { + o, err := makeOptions(repo, options...) if err != nil { return err } - scopes := scopesForUploadingImage(ref, []v1.Layer{layer}) - tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, scopes) + scopes := scopesForUploadingImage(repo, []v1.Layer{layer}) + tr, err := transport.New(repo.Registry, o.auth, o.transport, scopes) if err != nil { return err } w := writer{ - ref: ref, + repo: repo, client: &http.Client{Transport: tr}, } @@ -564,9 +564,9 @@ func Tag(tag name.Tag, t Taggable, options ...Option) error { return err } w := writer{ - ref: tag, + repo: tag.Context(), client: &http.Client{Transport: tr}, } - return w.commitImage(t) + return w.commitImage(t, tag) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go index 0d36125d6f..67a5d4379b 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/image.go @@ -79,15 +79,17 @@ func Image(opener Opener, tag *name.Tag) (v1.Image, error) { } // Peek at the first layer and see if it's compressed. - compressed, err := img.areLayersCompressed() - if err != nil { - return nil, err - } - if compressed { - c := compressedImage{ - image: img, + if len(img.imgDescriptor.Layers) > 0 { + compressed, err := img.areLayersCompressed() + if err != nil { + return nil, err + } + if compressed { + c := compressedImage{ + image: img, + } + return partial.CompressedToImage(&c) } - return partial.CompressedToImage(&c) } uc := uncompressedImage{ diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go index 314a51f140..23729af4f9 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/layer.go @@ -122,7 +122,7 @@ func LayerFromOpener(opener Opener, opts ...LayerOption) (v1.Layer, error) { } // LayerFromReader returns a v1.Layer given a io.Reader. -func LayerFromReader(reader io.Reader) (v1.Layer, error) { +func LayerFromReader(reader io.Reader, opts ...LayerOption) (v1.Layer, error) { // Buffering due to Opener requiring multiple calls. a, err := ioutil.ReadAll(reader) if err != nil { @@ -130,7 +130,7 @@ func LayerFromReader(reader io.Reader) (v1.Layer, error) { } return LayerFromOpener(func() (io.ReadCloser, error) { return ioutil.NopCloser(bytes.NewReader(a)), nil - }) + }, opts...) } func computeDigest(opener Opener, compressed bool, compression int) (v1.Hash, int64, error) { diff --git a/vendor/modules.txt b/vendor/modules.txt index 2e69b37fc1..48df743a0c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -84,7 +84,7 @@ github.com/google/go-cmp/cmp/internal/diff github.com/google/go-cmp/cmp/internal/flags github.com/google/go-cmp/cmp/internal/function github.com/google/go-cmp/cmp/internal/value -# github.com/google/go-containerregistry v0.0.0-20200115190719-e8e9aa676278 +# github.com/google/go-containerregistry v0.0.0-20200212224832-c629a66d7231 github.com/google/go-containerregistry/pkg/authn github.com/google/go-containerregistry/pkg/internal/retry github.com/google/go-containerregistry/pkg/internal/retry/wait