From 34e28747e26830af7a5a1ebe7d3e4de08ded6519 Mon Sep 17 00:00:00 2001 From: Urvashi Mohnani Date: Tue, 11 Sep 2018 16:24:15 -0400 Subject: [PATCH 1/2] Vendor in latest containers/image Picks up changes made to authentication for registry search. Signed-off-by: Urvashi Mohnani --- vendor.conf | 2 +- .../containers/image/docker/docker_client.go | 87 +++++++++++-------- .../containers/image/docker/docker_image.go | 2 +- .../image/docker/docker_image_dest.go | 12 +-- .../image/docker/docker_image_src.go | 10 +-- .../containers/image/ostree/ostree_dest.go | 4 +- 6 files changed, 68 insertions(+), 49 deletions(-) diff --git a/vendor.conf b/vendor.conf index 390615dfa8..c21cb4b8ec 100644 --- a/vendor.conf +++ b/vendor.conf @@ -10,7 +10,7 @@ github.com/containerd/cgroups 58556f5ad8448d99a6f7bea69ea4bdb7747cfeb0 github.com/containerd/continuity master github.com/containernetworking/cni v0.7.0-alpha1 github.com/containernetworking/plugins 1562a1e60ed101aacc5e08ed9dbeba8e9f3d4ec1 -github.com/containers/image 5df44e095ed826fbe2beeaabb329c749d7d6c3b6 +github.com/containers/image d8b5cf2b804a48489e5203d51254ef576794049d github.com/containers/storage 243c4cd616afdf06b4a975f18c4db083d26b1641 github.com/containers/psgo 5dde6da0bc8831b35243a847625bcf18183bd1ee github.com/coreos/go-systemd v14 diff --git a/vendor/github.com/containers/image/docker/docker_client.go b/vendor/github.com/containers/image/docker/docker_client.go index cae73a6cd7..4fb10c3958 100644 --- a/vendor/github.com/containers/image/docker/docker_client.go +++ b/vendor/github.com/containers/image/docker/docker_client.go @@ -100,6 +100,19 @@ type authScope struct { actions string } +// sendAuth determines whether we need authentication for v2 or v1 endpoint. +type sendAuth int + +const ( + // v2 endpoint with authentication. + v2Auth sendAuth = iota + // v1 endpoint with authentication. + // TODO: Get v1Auth working + // v1Auth + // no authentication, works for both v1 and v2. + noAuth +) + func newBearerTokenFromJSONBlob(blob []byte) (*bearerToken, error) { token := new(bearerToken) if err := json.Unmarshal(blob, &token); err != nil { @@ -234,7 +247,7 @@ func CheckAuth(ctx context.Context, sys *types.SystemContext, username, password return errors.Wrapf(err, "error creating new docker client") } - resp, err := newLoginClient.makeRequest(ctx, "GET", "/v2/", nil, nil) + resp, err := newLoginClient.makeRequest(ctx, "GET", "/v2/", nil, nil, v2Auth) if err != nil { return err } @@ -297,14 +310,43 @@ func SearchRegistry(ctx context.Context, sys *types.SystemContext, registry, ima return nil, errors.Wrapf(err, "error creating new docker client") } + // Only try the v1 search endpoint if the search query is not empty. If it is + // empty skip to the v2 endpoint. + if image != "" { + // set up the query values for the v1 endpoint + u := url.URL{ + Path: "/v1/search", + } + q := u.Query() + q.Set("q", image) + q.Set("n", strconv.Itoa(limit)) + u.RawQuery = q.Encode() + + logrus.Debugf("trying to talk to v1 search endpoint\n") + resp, err := client.makeRequest(ctx, "GET", u.String(), nil, nil, noAuth) + if err != nil { + logrus.Debugf("error getting search results from v1 endpoint %q: %v", registry, err) + } else { + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + logrus.Debugf("error getting search results from v1 endpoint %q, status code %d", registry, resp.StatusCode) + } else { + if err := json.NewDecoder(resp.Body).Decode(v1Res); err != nil { + return nil, err + } + return v1Res.Results, nil + } + } + } + logrus.Debugf("trying to talk to v2 search endpoint\n") - resp, err := client.makeRequest(ctx, "GET", "/v2/_catalog", nil, nil) + resp, err := client.makeRequest(ctx, "GET", "/v2/_catalog", nil, nil, v2Auth) if err != nil { logrus.Debugf("error getting search results from v2 endpoint %q: %v", registry, err) } else { defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - logrus.Debugf("error getting search results from v2 endpoint %q, status code %q", registry, resp.StatusCode) + logrus.Errorf("error getting search results from v2 endpoint %q, status code %d", registry, resp.StatusCode) } else { if err := json.NewDecoder(resp.Body).Decode(v2Res); err != nil { return nil, err @@ -322,50 +364,25 @@ func SearchRegistry(ctx context.Context, sys *types.SystemContext, registry, ima } } - // set up the query values for the v1 endpoint - u := url.URL{ - Path: "/v1/search", - } - q := u.Query() - q.Set("q", image) - q.Set("n", strconv.Itoa(limit)) - u.RawQuery = q.Encode() - - logrus.Debugf("trying to talk to v1 search endpoint\n") - resp, err = client.makeRequest(ctx, "GET", u.String(), nil, nil) - if err != nil { - logrus.Debugf("error getting search results from v1 endpoint %q: %v", registry, err) - } else { - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - logrus.Debugf("error getting search results from v1 endpoint %q, status code %q", registry, resp.StatusCode) - } else { - if err := json.NewDecoder(resp.Body).Decode(v1Res); err != nil { - return nil, err - } - return v1Res.Results, nil - } - } - return nil, errors.Wrapf(err, "couldn't search registry %q", registry) } // makeRequest creates and executes a http.Request with the specified parameters, adding authentication and TLS options for the Docker client. // The host name and schema is taken from the client or autodetected, and the path is relative to it, i.e. the path usually starts with /v2/. -func (c *dockerClient) makeRequest(ctx context.Context, method, path string, headers map[string][]string, stream io.Reader) (*http.Response, error) { +func (c *dockerClient) makeRequest(ctx context.Context, method, path string, headers map[string][]string, stream io.Reader, auth sendAuth) (*http.Response, error) { if err := c.detectProperties(ctx); err != nil { return nil, err } url := fmt.Sprintf("%s://%s%s", c.scheme, c.registry, path) - return c.makeRequestToResolvedURL(ctx, method, url, headers, stream, -1, true) + return c.makeRequestToResolvedURL(ctx, method, url, headers, stream, -1, auth) } // makeRequestToResolvedURL creates and executes a http.Request with the specified parameters, adding authentication and TLS options for the Docker client. // streamLen, if not -1, specifies the length of the data expected on stream. // makeRequest should generally be preferred. // TODO(runcom): too many arguments here, use a struct -func (c *dockerClient) makeRequestToResolvedURL(ctx context.Context, method, url string, headers map[string][]string, stream io.Reader, streamLen int64, sendAuth bool) (*http.Response, error) { +func (c *dockerClient) makeRequestToResolvedURL(ctx context.Context, method, url string, headers map[string][]string, stream io.Reader, streamLen int64, auth sendAuth) (*http.Response, error) { req, err := http.NewRequest(method, url, stream) if err != nil { return nil, err @@ -383,7 +400,7 @@ func (c *dockerClient) makeRequestToResolvedURL(ctx context.Context, method, url if c.sys != nil && c.sys.DockerRegistryUserAgent != "" { req.Header.Add("User-Agent", c.sys.DockerRegistryUserAgent) } - if sendAuth { + if auth == v2Auth { if err := c.setupRequestAuth(req); err != nil { return nil, err } @@ -497,7 +514,7 @@ func (c *dockerClient) detectProperties(ctx context.Context) error { ping := func(scheme string) error { url := fmt.Sprintf(resolvedPingV2URL, scheme, c.registry) - resp, err := c.makeRequestToResolvedURL(ctx, "GET", url, nil, nil, -1, true) + resp, err := c.makeRequestToResolvedURL(ctx, "GET", url, nil, nil, -1, noAuth) if err != nil { logrus.Debugf("Ping %s err %#v", url, err) return err @@ -524,7 +541,7 @@ func (c *dockerClient) detectProperties(ctx context.Context) error { // best effort to understand if we're talking to a V1 registry pingV1 := func(scheme string) bool { url := fmt.Sprintf(resolvedPingV1URL, scheme, c.registry) - resp, err := c.makeRequestToResolvedURL(ctx, "GET", url, nil, nil, -1, true) + resp, err := c.makeRequestToResolvedURL(ctx, "GET", url, nil, nil, -1, noAuth) logrus.Debugf("Ping %s err %#v", url, err) if err != nil { return false @@ -551,7 +568,7 @@ func (c *dockerClient) detectProperties(ctx context.Context) error { // using the original data structures. func (c *dockerClient) getExtensionsSignatures(ctx context.Context, ref dockerReference, manifestDigest digest.Digest) (*extensionSignatureList, error) { path := fmt.Sprintf(extensionsSignaturePath, reference.Path(ref.ref), manifestDigest) - res, err := c.makeRequest(ctx, "GET", path, nil, nil) + res, err := c.makeRequest(ctx, "GET", path, nil, nil, v2Auth) if err != nil { return nil, err } diff --git a/vendor/github.com/containers/image/docker/docker_image.go b/vendor/github.com/containers/image/docker/docker_image.go index 0107912417..a1a1150809 100644 --- a/vendor/github.com/containers/image/docker/docker_image.go +++ b/vendor/github.com/containers/image/docker/docker_image.go @@ -66,7 +66,7 @@ func GetRepositoryTags(ctx context.Context, sys *types.SystemContext, ref types. tags := make([]string, 0) for { - res, err := client.makeRequest(ctx, "GET", path, nil, nil) + res, err := client.makeRequest(ctx, "GET", path, nil, nil, v2Auth) if err != nil { return nil, err } diff --git a/vendor/github.com/containers/image/docker/docker_image_dest.go b/vendor/github.com/containers/image/docker/docker_image_dest.go index 85d5689952..94763d0266 100644 --- a/vendor/github.com/containers/image/docker/docker_image_dest.go +++ b/vendor/github.com/containers/image/docker/docker_image_dest.go @@ -130,7 +130,7 @@ func (d *dockerImageDestination) PutBlob(ctx context.Context, stream io.Reader, // FIXME? Chunked upload, progress reporting, etc. uploadPath := fmt.Sprintf(blobUploadPath, reference.Path(d.ref.ref)) logrus.Debugf("Uploading %s", uploadPath) - res, err := d.c.makeRequest(ctx, "POST", uploadPath, nil, nil) + res, err := d.c.makeRequest(ctx, "POST", uploadPath, nil, nil, v2Auth) if err != nil { return types.BlobInfo{}, err } @@ -147,7 +147,7 @@ func (d *dockerImageDestination) PutBlob(ctx context.Context, stream io.Reader, digester := digest.Canonical.Digester() sizeCounter := &sizeCounter{} tee := io.TeeReader(stream, io.MultiWriter(digester.Hash(), sizeCounter)) - res, err = d.c.makeRequestToResolvedURL(ctx, "PATCH", uploadLocation.String(), map[string][]string{"Content-Type": {"application/octet-stream"}}, tee, inputInfo.Size, true) + res, err = d.c.makeRequestToResolvedURL(ctx, "PATCH", uploadLocation.String(), map[string][]string{"Content-Type": {"application/octet-stream"}}, tee, inputInfo.Size, v2Auth) if err != nil { logrus.Debugf("Error uploading layer chunked, response %#v", res) return types.BlobInfo{}, err @@ -166,7 +166,7 @@ func (d *dockerImageDestination) PutBlob(ctx context.Context, stream io.Reader, // TODO: check inputInfo.Digest == computedDigest https://github.com/containers/image/pull/70#discussion_r77646717 locationQuery.Set("digest", computedDigest.String()) uploadLocation.RawQuery = locationQuery.Encode() - res, err = d.c.makeRequestToResolvedURL(ctx, "PUT", uploadLocation.String(), map[string][]string{"Content-Type": {"application/octet-stream"}}, nil, -1, true) + res, err = d.c.makeRequestToResolvedURL(ctx, "PUT", uploadLocation.String(), map[string][]string{"Content-Type": {"application/octet-stream"}}, nil, -1, v2Auth) if err != nil { return types.BlobInfo{}, err } @@ -191,7 +191,7 @@ func (d *dockerImageDestination) HasBlob(ctx context.Context, info types.BlobInf checkPath := fmt.Sprintf(blobsPath, reference.Path(d.ref.ref), info.Digest.String()) logrus.Debugf("Checking %s", checkPath) - res, err := d.c.makeRequest(ctx, "HEAD", checkPath, nil, nil) + res, err := d.c.makeRequest(ctx, "HEAD", checkPath, nil, nil, v2Auth) if err != nil { return false, -1, err } @@ -237,7 +237,7 @@ func (d *dockerImageDestination) PutManifest(ctx context.Context, m []byte) erro if mimeType != "" { headers["Content-Type"] = []string{mimeType} } - res, err := d.c.makeRequest(ctx, "PUT", path, headers, bytes.NewReader(m)) + res, err := d.c.makeRequest(ctx, "PUT", path, headers, bytes.NewReader(m), v2Auth) if err != nil { return err } @@ -442,7 +442,7 @@ sigExists: } path := fmt.Sprintf(extensionsSignaturePath, reference.Path(d.ref.ref), d.manifestDigest.String()) - res, err := d.c.makeRequest(ctx, "PUT", path, nil, bytes.NewReader(body)) + res, err := d.c.makeRequest(ctx, "PUT", path, nil, bytes.NewReader(body), v2Auth) if err != nil { return err } diff --git a/vendor/github.com/containers/image/docker/docker_image_src.go b/vendor/github.com/containers/image/docker/docker_image_src.go index 46145dff64..3ff826aaa7 100644 --- a/vendor/github.com/containers/image/docker/docker_image_src.go +++ b/vendor/github.com/containers/image/docker/docker_image_src.go @@ -89,7 +89,7 @@ func (s *dockerImageSource) fetchManifest(ctx context.Context, tagOrDigest strin path := fmt.Sprintf(manifestPath, reference.Path(s.ref.ref), tagOrDigest) headers := make(map[string][]string) headers["Accept"] = manifest.DefaultRequestedManifestMIMETypes - res, err := s.c.makeRequest(ctx, "GET", path, headers, nil) + res, err := s.c.makeRequest(ctx, "GET", path, headers, nil, v2Auth) if err != nil { return nil, "", err } @@ -137,7 +137,7 @@ func (s *dockerImageSource) getExternalBlob(ctx context.Context, urls []string) err error ) for _, url := range urls { - resp, err = s.c.makeRequestToResolvedURL(ctx, "GET", url, nil, nil, -1, false) + resp, err = s.c.makeRequestToResolvedURL(ctx, "GET", url, nil, nil, -1, noAuth) if err == nil { if resp.StatusCode != http.StatusOK { err = errors.Errorf("error fetching external blob from %q: %d", url, resp.StatusCode) @@ -169,7 +169,7 @@ func (s *dockerImageSource) GetBlob(ctx context.Context, info types.BlobInfo) (i path := fmt.Sprintf(blobsPath, reference.Path(s.ref.ref), info.Digest.String()) logrus.Debugf("Downloading %s", path) - res, err := s.c.makeRequest(ctx, "GET", path, nil, nil) + res, err := s.c.makeRequest(ctx, "GET", path, nil, nil, v2Auth) if err != nil { return nil, 0, err } @@ -332,7 +332,7 @@ func deleteImage(ctx context.Context, sys *types.SystemContext, ref dockerRefere return err } getPath := fmt.Sprintf(manifestPath, reference.Path(ref.ref), refTail) - get, err := c.makeRequest(ctx, "GET", getPath, headers, nil) + get, err := c.makeRequest(ctx, "GET", getPath, headers, nil, v2Auth) if err != nil { return err } @@ -354,7 +354,7 @@ func deleteImage(ctx context.Context, sys *types.SystemContext, ref dockerRefere // When retrieving the digest from a registry >= 2.3 use the following header: // "Accept": "application/vnd.docker.distribution.manifest.v2+json" - delete, err := c.makeRequest(ctx, "DELETE", deletePath, headers, nil) + delete, err := c.makeRequest(ctx, "DELETE", deletePath, headers, nil, v2Auth) if err != nil { return err } diff --git a/vendor/github.com/containers/image/ostree/ostree_dest.go b/vendor/github.com/containers/image/ostree/ostree_dest.go index 0ae30f1f4d..afff7dc1b5 100644 --- a/vendor/github.com/containers/image/ostree/ostree_dest.go +++ b/vendor/github.com/containers/image/ostree/ostree_dest.go @@ -467,7 +467,9 @@ func (d *ostreeImageDestination) Commit(ctx context.Context) error { metadata := []string{fmt.Sprintf("docker.manifest=%s", string(d.manifest)), fmt.Sprintf("signatures=%d", d.signaturesLen), fmt.Sprintf("docker.digest=%s", string(d.digest))} - err = d.ostreeCommit(repo, fmt.Sprintf("ociimage/%s", d.ref.branchName), manifestPath, metadata) + if err := d.ostreeCommit(repo, fmt.Sprintf("ociimage/%s", d.ref.branchName), manifestPath, metadata); err != nil { + return err + } _, err = repo.CommitTransaction() return err From bbcfa3b2e4bd369ca65e4130632ffe5888f1058d Mon Sep 17 00:00:00 2001 From: Urvashi Mohnani Date: Tue, 11 Sep 2018 16:37:43 -0400 Subject: [PATCH 2/2] Search registries with an empty query Adds functionality to search registries implementing the v2 endpoint with an empty query, that is the results will be all the available images on the registries. If this is tried with a v1 registry an error will occur. To search a whole registry, there needs to be a trailing slash at the end, i.e `podman search registry.fedoraproject.org/`. Signed-off-by: Urvashi Mohnani --- cmd/podman/search.go | 5 +++++ docs/podman-search.1.md | 27 +++++++++++++++++++++++++-- test/e2e/search_test.go | 9 ++++++++- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/cmd/podman/search.go b/cmd/podman/search.go index 009ff8ba97..f64b822fc5 100644 --- a/cmd/podman/search.go +++ b/cmd/podman/search.go @@ -345,6 +345,11 @@ func matchesOfficialFilter(filter searchFilterParams, result docker.SearchResult } func getRegistry(image string) (string, error) { + // It is possible to only have the registry name in the format "myregistry/" + // if so, just trim the "/" from the end and return the registry name + if strings.HasSuffix(image, "/") { + return strings.TrimSuffix(image, "/"), nil + } imgRef, err := reference.Parse(image) if err != nil { return "", err diff --git a/docs/podman-search.1.md b/docs/podman-search.1.md index 9cc9e1e6bf..429c3c5ad2 100644 --- a/docs/podman-search.1.md +++ b/docs/podman-search.1.md @@ -10,10 +10,12 @@ podman\-search - Search a registry for an image **podman search** searches a registry or a list of registries for a matching image. The user can specify which registry to search by prefixing the registry in the search term (example **registry.fedoraproject.org/fedora**), default is the registries in the -**registires.search** table in the config file - **/etc/containers/registries.conf**. +**registries.search** table in the config file - **/etc/containers/registries.conf**. The number of results can be limited using the **--limit** flag. If more than one registry is being searched, the limit will be applied to each registry. The output can be filtered -using the **--filter** flag. +using the **--filter** flag. To get all available images in a registry without a specific +search term, the user can just enter the registry name with a trailing "/" (example **registry.fedoraproject.org/**). +Note, searching without a search term will only work for registries that implement the v2 API. **podman [GLOBAL OPTIONS]** @@ -116,6 +118,27 @@ INDEX NAME fedoraproject.org fedoraproject.org/fedora fedoraproject.org fedoraproject.org/fedora-minimal ``` + +``` +$ podman search registry.fedoraproject.org/ +INDEX NAME DESCRIPTION STARS OFFICIAL AUTOMATED +fedoraproject.org registry.fedoraproject.org/f25/cockpit 0 +fedoraproject.org registry.fedoraproject.org/f25/container-engine 0 +fedoraproject.org registry.fedoraproject.org/f25/docker 0 +fedoraproject.org registry.fedoraproject.org/f25/etcd 0 +fedoraproject.org registry.fedoraproject.org/f25/flannel 0 +fedoraproject.org registry.fedoraproject.org/f25/httpd 0 +fedoraproject.org registry.fedoraproject.org/f25/kubernetes-apiserver 0 +fedoraproject.org registry.fedoraproject.org/f25/kubernetes-controller-manager 0 +fedoraproject.org registry.fedoraproject.org/f25/kubernetes-kubelet 0 +fedoraproject.org registry.fedoraproject.org/f25/kubernetes-master 0 +fedoraproject.org registry.fedoraproject.org/f25/kubernetes-node 0 +fedoraproject.org registry.fedoraproject.org/f25/kubernetes-proxy 0 +fedoraproject.org registry.fedoraproject.org/f25/kubernetes-scheduler 0 +fedoraproject.org registry.fedoraproject.org/f25/mariadb 0 +``` +Note: This works only with registries that implement the v2 API. If tried with a v1 registry an error will be returned. + ## FILES **registries.conf** (`/etc/containers/registries.conf`) diff --git a/test/e2e/search_test.go b/test/e2e/search_test.go index 7b9612a35a..2c85ca7656 100644 --- a/test/e2e/search_test.go +++ b/test/e2e/search_test.go @@ -84,7 +84,7 @@ var _ = Describe("Podman search", func() { }) It("podman search limit flag", func() { - search := podmanTest.Podman([]string{"search", "--limit", "3", "alpine"}) + search := podmanTest.Podman([]string{"search", "--limit", "3", "docker.io/alpine"}) search.WaitWithDefaultTimeout() Expect(search.ExitCode()).To(Equal(0)) Expect(len(search.OutputToStringArray())).To(Equal(4)) @@ -120,6 +120,13 @@ var _ = Describe("Podman search", func() { } }) + It("podman search v2 registry with empty query", func() { + search := podmanTest.Podman([]string{"search", "registry.fedoraproject.org/"}) + search.WaitWithDefaultTimeout() + Expect(search.ExitCode()).To(Equal(0)) + Expect(len(search.OutputToStringArray())).To(BeNumerically(">=", 1)) + }) + It("podman search attempts HTTP if tls-verify flag is set false", func() { podmanTest.RestoreArtifact(registry) fakereg := podmanTest.Podman([]string{"run", "-d", "--name", "registry", "-p", "5000:5000", registry, "/entrypoint.sh", "/etc/docker/registry/config.yml"})