represent all content as oci layouts (artifact.OCI interface), ad…

…d blob caching and ephemeral stores (#59)

* represent all content as artifact.OCI interface and manipulate/add all content using oci layouts
* initial brew taps and macos universal binary
* change mediaType to string for better compatibility with other libraries
* ensure config is minimally viable for file/charts
* add transparent layer caching (filesystem) to artifact operations, clean up layer interface used by file/chart
* add store list and store copy commands

Signed-off-by: Josh Wolf <josh@joshwolf.dev>

.gitignore

@@ -27,3 +27,4 @@ dist/
 tmp/
 bin/
 pkg.yaml
+store/

.goreleaser.yaml

@@ -4,13 +4,21 @@ builds:
     goos:
       - linux
       - darwin
+      - windows
     goarch:
       - amd64
       - arm64
     env:
       - CGO_ENABLED=0
-#    flags:
-#      - -tags=containers_image_openpgp containers_image_ostree
-#release:
-#  extra_files:
-#    - glob: ./pkg.tar.zst
+
+universal_binaries:
+  - replace: true
+
+brews:
+  - name: hauler
+    tap:
+      owner: rancherfederal
+      name: homebrew-tap
+      token: "{{ .Env.HOMEBREW_TAP_GITHUB_TOKEN }}"
+    folder: Formula
+    description: "Hauler CLI"

cmd/hauler/cli/cli.go

@@ -2,20 +2,21 @@ package cli
 
 import (
 	"context"
-	"fmt"
+	"errors"
 	"os"
 	"path/filepath"
 
 	"github.com/spf13/cobra"
 
+	"github.com/rancherfederal/hauler/pkg/cache"
 	"github.com/rancherfederal/hauler/pkg/log"
 	"github.com/rancherfederal/hauler/pkg/store"
 )
 
 type rootOpts struct {
 	logLevel string
-	dataDir  string
 	cacheDir string
+	storeDir string
 }
 
 var ro = &rootOpts{}
@@ -35,8 +36,8 @@ func New() *cobra.Command {
 
 	pf := cmd.PersistentFlags()
 	pf.StringVarP(&ro.logLevel, "log-level", "l", "info", "")
-	pf.StringVar(&ro.dataDir, "content-dir", "", "Location of where to create and store contents (defaults to ~/.local/hauler)")
-	pf.StringVar(&ro.cacheDir, "cache", "", "Location of where to store cache data (defaults to XDG_CACHE_DIR/hauler)")
+	pf.StringVar(&ro.cacheDir, "cache", "", "Location of where to store cache data (defaults to $XDG_CACHE_DIR/hauler)")
+	pf.StringVarP(&ro.storeDir, "store", "s", "", "Location to create store at (defaults to $PWD/store)")
 
 	// Add subcommands
 	addDownload(cmd)
@@ -46,37 +47,56 @@ func New() *cobra.Command {
 }
 
 func (o *rootOpts) getStore(ctx context.Context) (*store.Store, error) {
-	dir := o.dataDir
+	lgr := log.FromContext(ctx)
+	dir := o.storeDir
 
-	if o.dataDir == "" {
-		// Default to userspace
-		home, err := os.UserHomeDir()
+	if dir == "" {
+		lgr.Debugf("no store path specified, defaulting to $PWD/store")
+		pwd, err := os.Getwd()
 		if err != nil {
 			return nil, err
 		}
 
-		abs, _ := filepath.Abs(filepath.Join(home, ".local/hauler/store"))
-		if err := os.MkdirAll(abs, os.ModePerm); err != nil {
+		dir = filepath.Join(pwd, "store")
+	}
+
+	abs, err := filepath.Abs(dir)
+	if err != nil {
+		return nil, err
+	}
+
+	lgr.Debugf("using store at %s", abs)
+	if _, err := os.Stat(abs); errors.Is(err, os.ErrNotExist) {
+		err := os.Mkdir(abs, os.ModePerm)
+		if err != nil {
 			return nil, err
 		}
+	} else if err != nil {
+		return nil, err
+	}
 
-		dir = abs
-	} else {
-		// Make sure directory exists and we can write to it
-		if f, err := os.Stat(o.dataDir); err != nil {
-			return nil, err
-		} else if !f.IsDir() {
-			return nil, fmt.Errorf("%s is not a directory", o.dataDir)
-		} // TODO: Add writeable check
+	s := store.NewStore(ctx, abs)
+	return s, nil
+}
 
-		abs, err := filepath.Abs(o.dataDir)
+func (o *rootOpts) getCache(ctx context.Context) (cache.Cache, error) {
+	dir := o.cacheDir
+
+	if dir == "" {
+		// Default to $XDG_CACHE_DIR
+		cachedir, err := os.UserCacheDir()
 		if err != nil {
 			return nil, err
 		}
 
+		abs, _ := filepath.Abs(filepath.Join(cachedir, "hauler"))
+		if err := os.MkdirAll(abs, os.ModePerm); err != nil {
+			return nil, err
+		}
+
 		dir = abs
 	}
 
-	s := store.NewStore(ctx, dir)
-	return s, nil
+	c := cache.NewFilesystem(dir)
+	return c, nil
 }

cmd/hauler/cli/download.go

@@ -10,8 +10,24 @@ func addDownload(parent *cobra.Command) {
 	o := &download.Opts{}
 
 	cmd := &cobra.Command{
-		Use:     "download",
-		Short:   "Download OCI content from a registry and populate it on disk",
+		Use:   "download",
+		Short: "Download OCI content from a registry and populate it on disk",
+		Long: `Locate OCI content based on it's reference in a compatible registry and download the contents to disk.
+
+Note that the content type determines it's format on disk.  Hauler's built in content types act as follows:
+
+	- File: as a file named after the pushed contents source name (ex: hauler/my-file.yaml:latest --> my-file.yaml)
+	- Image: as a .tar named after the image (ex: alpine:latest --> alpine:latest.tar)
+	- Chart: as a .tar.gz named after the chart (ex: grafana/loki:2.0.2 --> grafana-loki-2.0.2.tar.gz)`,
+		Example: `
+# Download a file
+hauler dl hauler/my-file.yaml:latest
+
+# Download an image
+hauler dl rancher/k3s:v1.22.2-k3s2
+
+# Download a chart
+hauler dl hauler/longhorn:1.2.0`,
 		Aliases: []string{"dl"},
 		Args:    cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, arg []string) error {

cmd/hauler/cli/download/download.go

@@ -2,34 +2,39 @@ package download
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"path"
 
-	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/remotes/docker"
 	"github.com/google/go-containerregistry/pkg/authn"
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/google/go-containerregistry/pkg/v1/tarball"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/spf13/cobra"
 	"oras.land/oras-go/pkg/content"
 	"oras.land/oras-go/pkg/oras"
 
+	"github.com/rancherfederal/hauler/pkg/artifact/types"
 	"github.com/rancherfederal/hauler/pkg/log"
 )
 
 type Opts struct {
 	DestinationDir string
+	OutputFile     string
 }
 
 func (o *Opts) AddArgs(cmd *cobra.Command) {
 	f := cmd.Flags()
 
 	f.StringVar(&o.DestinationDir, "dir", "", "Directory to save contents to (defaults to current directory)")
+	f.StringVarP(&o.OutputFile, "output", "o", "", "(Optional) Override name of file to save.")
 }
 
 func Cmd(ctx context.Context, o *Opts, reference string) error {
-	l := log.FromContext(ctx)
-	l.Debugf("running command `hauler download`")
+	lgr := log.FromContext(ctx)
+	lgr.Debugf("running command `hauler download`")
 
 	cs := content.NewFileStore(o.DestinationDir)
 	defer cs.Close()
@@ -39,36 +44,86 @@ func Cmd(ctx context.Context, o *Opts, reference string) error {
 		return err
 	}
 
-	resolver := docker.NewResolver(docker.ResolverOptions{})
+	// resolver := docker.NewResolver(docker.ResolverOptions{})
 
 	desc, err := remote.Get(ref)
 	if err != nil {
 		return err
 	}
 
-	l.Debugf("Getting content of media type: %s", desc.MediaType)
-	switch desc.MediaType {
-	case ocispec.MediaTypeImageManifest:
-		desc, artifacts, err := oras.Pull(ctx, resolver, ref.Name(), cs, oras.WithPullBaseHandler())
+	manifestData, err := desc.RawManifest()
+	if err != nil {
+		return err
+	}
+
+	var manifest ocispec.Manifest
+	if err := json.Unmarshal(manifestData, &manifest); err != nil {
+		return err
+	}
+
+	// TODO: These need to be factored out into each of the contents own logic
+	switch manifest.Config.MediaType {
+	case types.DockerConfigJSON, types.OCIManifestSchema1:
+		lgr.Infof("identified [image] (%s) content", manifest.Config.MediaType)
+		img, err := remote.Image(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain))
 		if err != nil {
 			return err
 		}
 
-		// TODO: Better logging
-		_ = desc
-		_ = artifacts
-		// l.Infof("Downloaded %d artifacts: %s", len(artifacts), content.ResolveName(desc))
+		outputFile := o.OutputFile
+		if outputFile == "" {
+			outputFile = fmt.Sprintf("%s:%s.tar", path.Base(ref.Context().RepositoryStr()), ref.Identifier())
+		}
 
-	case images.MediaTypeDockerSchema2Manifest:
-		img, err := remote.Image(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain))
+		if err := tarball.WriteToFile(outputFile, ref, img); err != nil {
+			return err
+		}
+
+		lgr.Infof("downloaded [%s] to [%s]", ref.Name(), outputFile)
+
+	case types.FileMediaType:
+		lgr.Infof("identified [file] (%s) content", manifest.Config.MediaType)
+
+		fs := content.NewFileStore(o.DestinationDir)
+
+		resolver := docker.NewResolver(docker.ResolverOptions{})
+		mdesc, descs, err := oras.Pull(ctx, resolver, ref.Name(), fs)
 		if err != nil {
 			return err
 		}
 
-		_ = img
+		lgr.Infof("downloaded [%d] files with digest [%s]", len(descs), mdesc)
+
+	case types.ChartLayerMediaType, types.ChartConfigMediaType:
+		lgr.Infof("identified [chart] (%s) content", manifest.Config.MediaType)
+
+		fs := content.NewFileStore(o.DestinationDir)
+
+		resolver := docker.NewResolver(docker.ResolverOptions{})
+		mdesc, _, err := oras.Pull(ctx, resolver, ref.Name(), fs)
+		if err != nil {
+			return err
+		}
+
+		lgr.Infof("downloaded chart [%s] with digest [%s]", "donkey", mdesc.Digest.String())
+
 	default:
-		return fmt.Errorf("unknown media type: %s", desc.MediaType)
+		return fmt.Errorf("unrecognized content type: %s", manifest.Config.MediaType)
 	}
 
 	return nil
 }
+
+func getManifest(ctx context.Context, ref string) (*remote.Descriptor, error) {
+	r, err := name.ParseReference(ref)
+	if err != nil {
+		return nil, fmt.Errorf("parsing reference %q: %v", ref, err)
+	}
+
+	desc, err := remote.Get(r, remote.WithContext(ctx))
+	if err != nil {
+		return nil, err
+	}
+
+	return desc, nil
+}

cmd/hauler/cli/download/download_test.go

@@ -0,0 +1,38 @@
+package download
+
+import (
+	"context"
+	"testing"
+)
+
+func TestCmd(t *testing.T) {
+	ctx := context.Background()
+
+	type args struct {
+		ctx       context.Context
+		o         *Opts
+		reference string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "should work",
+			args: args{
+				ctx:       ctx,
+				o:         &Opts{DestinationDir: ""},
+				reference: "localhost:3000/hauler/file.txt:latest",
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := Cmd(tt.args.ctx, tt.args.o, tt.args.reference); (err != nil) != tt.wantErr {
+				t.Errorf("Cmd() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}