Skip to content

Commit

Permalink
Refactor pull client code into client.go
Browse files Browse the repository at this point in the history
  • Loading branch information
hinshun committed Apr 2, 2019
1 parent 43b05cf commit 63fc57b
Show file tree
Hide file tree
Showing 6 changed files with 255 additions and 189 deletions.
107 changes: 107 additions & 0 deletions client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
package ipcs

import (
"context"

"github.com/containerd/containerd"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/remotes"
iface "github.com/ipfs/interface-go-ipfs-core"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)

type Client struct {
ipfsCln iface.CoreAPI
ctrdCln *containerd.Client
ipcs *store
}

func NewClient(ipfsCln iface.CoreAPI, ctrdCln *containerd.Client) *Client {
return &Client{
ipfsCln: ipfsCln,
ctrdCln: ctrdCln,
ipcs: &store{
cln: ipfsCln,
},
}
}

func (c *Client) Pull(ctx context.Context, ref string, desc ocispec.Descriptor) (containerd.Image, error) {
ctx, done, err := c.ctrdCln.WithLease(ctx)
if err != nil {
return nil, err
}
defer done(ctx)

img, err := c.Fetch(ctx, ref, desc)
if err != nil {
return nil, err
}

i := containerd.NewImageWithPlatform(c.ctrdCln, img, platforms.Default())

if err := i.Unpack(ctx, containerd.DefaultSnapshotter); err != nil {
return nil, errors.Wrapf(err, "failed to unpack image on snapshotter %s", containerd.DefaultSnapshotter)
}

return i, nil
}

func (c *Client) Fetch(ctx context.Context, ref string, desc ocispec.Descriptor) (images.Image, error) {
store := c.ctrdCln.ContentStore()
fetcher := c.ipcs

// Get all the children for a descriptor
childrenHandler := images.ChildrenHandler(store)
// Set any children labels for that content
childrenHandler = images.SetChildrenLabels(store, childrenHandler)
// Filter children by platforms
childrenHandler = images.FilterPlatforms(childrenHandler, platforms.Default())
// Sort and limit manifests if a finite number is needed
childrenHandler = images.LimitManifests(childrenHandler, platforms.Default(), 1)

handler := images.Handlers(
remotes.FetchHandler(store, fetcher),
childrenHandler,
)

if err := images.Dispatch(ctx, handler, desc); err != nil {
return images.Image{}, err
}

img := images.Image{
Name: ref,
Target: desc,
}

is := c.ctrdCln.ImageService()
for {
if created, err := is.Create(ctx, img); err != nil {
if !errdefs.IsAlreadyExists(err) {
return images.Image{}, err
}

updated, err := is.Update(ctx, img)
if err != nil {
// if image was removed, try create again
if errdefs.IsNotFound(err) {
continue
}
return images.Image{}, err
}

img = updated
} else {
img = created
}

return img, nil
}
}

func (c *Client) Push(ctx context.Context, ref string, desc ocispec.Descriptor) error {
return nil
}
163 changes: 4 additions & 159 deletions cmd/convert/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,33 +2,20 @@ package main

import (
"context"
"io"
"io/ioutil"
"log"
"net/http"
"net/url"
"os"
"path"
"strings"

"github.com/containerd/containerd"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/namespaces"
"github.com/containerd/containerd/oci"
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/reference"
"github.com/containerd/containerd/remotes/docker"
"github.com/hinshun/ipcs"
"github.com/hinshun/ipcs/digestconv"
httpapi "github.com/ipfs/go-ipfs-http-client"
iface "github.com/ipfs/interface-go-ipfs-core"
"github.com/moby/buildkit/util/contentutil"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/pkg/errors"
"golang.org/x/net/context/ctxhttp"
)

func main() {
Expand Down Expand Up @@ -89,11 +76,13 @@ func Convert(ctx context.Context, ipfsCln iface.CoreAPI, ctrdCln *containerd.Cli
return errors.Wrapf(err, "failed to convert %q to ipfs manifest", srcName)
}

err = PullByDescriptor(ctx, ipfsCln, ctrdCln, dst, mfstDesc)
ipcsCln := ipcs.NewClient(ipfsCln, ctrdCln)
img, err := ipcsCln.Pull(ctx, dst, mfstDesc)
if err != nil {
return errors.Wrapf(err, "failed to pull descriptor %q", mfstDesc.Digest)
}

log.Printf("Successfully pulled image %q", img.Name())
return nil
}

Expand All @@ -118,7 +107,7 @@ func RunContainer(ctx context.Context, cln *containerd.Client, ref, id string) e
)
cOpts = append(cOpts,
containerd.WithImage(image),
containerd.WithSnapshotter("native"),
containerd.WithSnapshotter(containerd.DefaultSnapshotter),
// Even when "readonly" is set, we don't use KindView snapshot here. (#1495)
// We pass writable snapshot to the OCI runtime, and the runtime remounts it as read-only,
// after creating some mount points on demand.
Expand All @@ -135,147 +124,3 @@ func RunContainer(ctx context.Context, cln *containerd.Client, ref, id string) e

return nil
}

func Pull(ctx context.Context, ipfsCln iface.CoreAPI, ctrdCln *containerd.Client, ref string) error {
// resolver := docker.NewResolver(docker.ResolverOptions{
// Client: http.DefaultClient,
// })

// name, desc, err := resolver.Resolve(ctx, ref)
// if err != nil {
// return errors.Wrapf(err, "failed to resolve %q", ref)
// }

name := ref
desc := ocispec.Descriptor{
MediaType: ocispec.MediaTypeImageManifest,
Digest: digest.Digest("sha256:c0ecc9c9fb27ebc68af1014d2f1962f1533fe606f2ed5963c61d066976ce8d5a"),
Size: 456,
}

return PullByDescriptor(ctx, ipfsCln, ctrdCln, name, desc)
}

func PullByDescriptor(ctx context.Context, ipfsCln iface.CoreAPI, ctrdCln *containerd.Client, name string, desc ocispec.Descriptor) error {
ingester := ctrdCln.ContentStore()
provider, err := ipcs.NewContentStore(ipcs.Config{
IpfsPath: "./tmp/ipfs",
})
if err != nil {
return errors.Wrap(err, "failed to create ipcs")
}

mfst, err := images.Manifest(ctx, provider, desc, platforms.Default())
if err != nil {
return errors.Wrap(err, "failed to get manifest")
}

err = contentutil.Copy(ctx, ingester, provider, mfst.Config)
if err != nil {
return errors.Wrapf(err, "failed to ingest manifest config blob %q", mfst.Config.Digest)
}

for _, layer := range mfst.Layers {
err = contentutil.Copy(ctx, ingester, provider, layer)
if err != nil {
return errors.Wrapf(err, "failed to ingest blob %q", layer.Digest)
}
}

err = contentutil.Copy(ctx, ingester, provider, desc)
if err != nil {
return errors.Wrapf(err, "failed to ingest manifest blob %q", desc.Digest)
}

image := images.Image{
Name: name,
Target: desc,
}

image, err = createImage(ctx, ctrdCln, image)
if err != nil {
return errors.Wrapf(err, "failed to create image %q", image.Name)
}

c, err := digestconv.DigestToCid(image.Target.Digest)
if err != nil {
return errors.Wrapf(err, "failed to convert image digest %q to cid", image.Target.Digest)
}
log.Printf("Successfully created image %q (%s)", image.Target.Digest, c)

p := []ocispec.Platform{platforms.DefaultSpec()}
for _, platform := range p {
log.Printf("Unpacking %q %q...\n", platforms.Format(platform), image.Target.Digest)
i := containerd.NewImageWithPlatform(ctrdCln, image, platforms.Only(platform))
err = i.Unpack(ctx, "native")
if err != nil {
return errors.Wrap(err, "failed to unpack image")
}
}

return nil
}

func createImage(ctx context.Context, cln *containerd.Client, img images.Image) (images.Image, error) {
is := cln.ImageService()
for {
if created, err := is.Create(ctx, img); err != nil {
if !errdefs.IsAlreadyExists(err) {
return images.Image{}, err
}

updated, err := is.Update(ctx, img)
if err != nil {
// if image was removed, try create again
if errdefs.IsNotFound(err) {
continue
}
return images.Image{}, err
}

img = updated
} else {
img = created
}

return img, nil
}
}

func pushTag(ctx context.Context, cln *http.Client, r io.Reader, ref string, desc ocispec.Descriptor) error {
refspec, err := reference.Parse(ref)
if err != nil {
return errors.Wrapf(err, "failed to parse reference %q", ref)
}

u := url.URL{
Host: refspec.Hostname(),
Scheme: "https",
}
if strings.HasPrefix(u.Host, "localhost:") {
u.Scheme = "http"
}
prefix := strings.TrimPrefix(refspec.Locator, u.Host+"/")
u.Path = path.Join("/v2", prefix, "manifests", refspec.Object)

req, err := http.NewRequest(http.MethodPut, u.String(), nil)
if err != nil {
return errors.Wrap(err, "failed to create http requestname, ")
}

req.Header.Add("Content-Type", desc.MediaType)
req.Body = ioutil.NopCloser(r)
req.ContentLength = desc.Size

resp, err := ctxhttp.Do(ctx, cln, req)
if err != nil {
return errors.Wrap(err, "failed to do request")
}

switch resp.StatusCode {
case http.StatusOK, http.StatusCreated, http.StatusNoContent:
return nil
default:
return errors.Wrapf(err, "failed to do request with status %q", resp.Status)
}
}
6 changes: 3 additions & 3 deletions converter.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,13 @@ func (c *converter) Convert(ctx context.Context, desc ocispec.Descriptor) (ocisp
if err != nil {
return ocispec.Descriptor{}, errors.Wrap(err, "failed to marshal manifest JSON")
}
log.Printf("Original Manifest [%d]:\n%s", len(origMfstJSON), origMfstJSON)
log.Printf("Original Manifest [%d] %s:\n%s", len(origMfstJSON), desc.Digest, origMfstJSON)

origMfstConfigJSON, err := content.ReadBlob(ctx, c.provider, mfst.Config)
if err != nil {
return ocispec.Descriptor{}, errors.Wrap(err, "failed to get original manifest config JSON")
}
log.Printf("Original Manifest Config [%d]:\n%s", len(origMfstConfigJSON), origMfstConfigJSON)
log.Printf("Original Manifest Config [%d] %s:\n%s", len(origMfstConfigJSON), mfst.Config.Digest, origMfstConfigJSON)

mfst.Config.Digest, err = copyFile(ctx, c.cln, c.provider, mfst.Config)
if err != nil {
Expand All @@ -67,12 +67,12 @@ func (c *converter) Convert(ctx context.Context, desc ocispec.Descriptor) (ocisp
if err != nil {
return ocispec.Descriptor{}, errors.Wrap(err, "failed to marshal manifest JSON")
}
log.Printf("Converted Manifest [%d]:\n%s", len(mfstJSON), mfstJSON)

mfstDigest, err := addFile(ctx, c.cln, files.NewBytesFile(mfstJSON))
if err != nil {
return ocispec.Descriptor{}, errors.Wrap(err, "failed to upload manifest")
}
log.Printf("Converted Manifest [%d] %s:\n%s", len(mfstJSON), mfstDigest, mfstJSON)

return ocispec.Descriptor{
MediaType: ocispec.MediaTypeImageManifest,
Expand Down
20 changes: 12 additions & 8 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@ go 1.12
require (
github.com/Microsoft/go-winio v0.4.12 // indirect
github.com/Microsoft/hcsshim v0.8.6 // indirect
github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect
github.com/btcsuite/btcd v0.0.0-20190315201642-aa6e0f35703c // indirect
github.com/containerd/aufs v0.0.0-20190114185352-f894a800659b
github.com/containerd/cgroups v0.0.0-20190226200435-dbea6f2bd416 // indirect
github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50 // indirect
github.com/containerd/containerd v1.2.4
github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 // indirect
github.com/containerd/cri v1.11.1 // indirect
github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448 // indirect
github.com/containerd/go-runc v0.0.0-20190226155025-7d11b49dc076 // indirect
github.com/containerd/ttrpc v0.0.0-20190211042230-69144327078c // indirect
Expand All @@ -24,16 +25,23 @@ require (
github.com/docker/go-units v0.3.3 // indirect
github.com/godbus/dbus v4.1.0+incompatible // indirect
github.com/gogo/googleapis v1.1.0 // indirect
github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect
github.com/google/go-cmp v0.2.0 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/ipfs/go-cid v0.0.1
github.com/ipfs/go-ipfs-files v0.0.1
github.com/ipfs/go-ipfs-http-client v0.0.1
github.com/ipfs/go-ipfs-util v0.0.1
github.com/ipfs/interface-go-ipfs-core v0.0.1
github.com/libp2p/go-flow-metrics v0.2.0 // indirect
github.com/libp2p/go-libp2p-crypto v2.0.1+incompatible // indirect
github.com/libp2p/go-libp2p-metrics v2.1.7+incompatible // indirect
github.com/libp2p/go-libp2p-peer v2.4.0+incompatible // indirect
github.com/libp2p/go-libp2p-protocol v1.0.0 // indirect
github.com/mistifyio/go-zfs v2.1.1+incompatible // indirect
github.com/moby/buildkit v0.3.3
github.com/multiformats/go-multihash v0.0.1
github.com/multiformats/go-multiaddr v0.0.2 // indirect
github.com/multiformats/go-multiaddr-net v1.6.3 // indirect
github.com/multiformats/go-multihash v1.0.8
github.com/opencontainers/go-digest v1.0.0-rc1
github.com/opencontainers/image-spec v1.0.1
github.com/opencontainers/runc v0.1.1 // indirect
Expand All @@ -44,10 +52,6 @@ require (
github.com/stretchr/testify v1.3.0
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 // indirect
go.etcd.io/bbolt v1.3.2 // indirect
golang.org/x/net v0.0.0-20190227160552-c95aed5357e7
google.golang.org/grpc v1.19.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
k8s.io/apimachinery v0.0.0-20190313115320-c9defaaddf6f // indirect
k8s.io/klog v0.2.0 // indirect
k8s.io/kubernetes v1.13.4 // indirect
gotest.tools v2.2.0+incompatible // indirect
)
Loading

0 comments on commit 63fc57b

Please sign in to comment.