From 8ab9fd6a38cf8e83d1977846e153adfdec27a00c Mon Sep 17 00:00:00 2001 From: Josh Wolf Date: Wed, 10 Nov 2021 10:37:21 -0700 Subject: [PATCH] represent all `content` as oci layouts (`artifact.OCI` interface), add blob caching and ephemeral stores (#59) * represent all content as artifact.OCI interface and manipulate/add all content using oci layouts * initial brew taps and macos universal binary * change mediaType to string for better compatibility with other libraries * ensure config is minimally viable for file/charts * add transparent layer caching (filesystem) to artifact operations, clean up layer interface used by file/chart * add store list and store copy commands Signed-off-by: Josh Wolf --- .gitignore | 1 + .goreleaser.yaml | 18 +- cmd/hauler/cli/cli.go | 62 ++++--- cmd/hauler/cli/download.go | 20 +- cmd/hauler/cli/download/download.go | 87 +++++++-- cmd/hauler/cli/download/download_test.go | 38 ++++ cmd/hauler/cli/store.go | 78 +++++++- cmd/hauler/cli/store/add.go | 129 +++++++++++-- cmd/hauler/cli/store/sync.go | 47 ++--- go.mod | 5 +- go.sum | 7 - pkg/artifact/config.go | 10 + pkg/artifact/local/layer.go | 127 +++++++++++++ pkg/artifact/oci.go | 20 ++ pkg/artifact/types/types.go | 26 +++ pkg/cache/cache.go | 102 ++++++++++ pkg/cache/doc.go | 5 + pkg/cache/filesystem.go | 120 ++++++++++++ .../k3s/dependencies.go | 0 pkg/collection/k3s/k3s.go | 46 +++++ pkg/{content => collection}/k3s/k3s_test.go | 0 pkg/content/chart/chart.go | 170 +++++++++-------- pkg/content/chart/chart_test.go | 29 ++- pkg/content/chart/dependents.go | 32 +++- pkg/content/content.go | 95 +--------- pkg/content/file/config.go | 82 +++++++++ pkg/content/file/file.go | 174 +++++++----------- pkg/content/file/file_test.go | 86 +++++++-- pkg/content/image/image.go | 49 +++-- pkg/content/image/image_test.go | 44 ++++- pkg/content/k3s/k3s.go | 53 ------ pkg/layout/artifact.go | 132 +++++++++++++ pkg/layout/store.go | 165 +++++++++++++++++ pkg/store/add.go | 91 ++++++--- pkg/store/store.go | 96 +++++++--- testdata/contents.yaml | 28 +-- 36 files changed, 1688 insertions(+), 586 deletions(-) create mode 100644 cmd/hauler/cli/download/download_test.go create mode 100644 pkg/artifact/config.go create mode 100644 pkg/artifact/local/layer.go create mode 100644 pkg/artifact/oci.go create mode 100644 pkg/artifact/types/types.go create mode 100644 pkg/cache/cache.go create mode 100644 pkg/cache/doc.go create mode 100644 pkg/cache/filesystem.go rename pkg/{content => collection}/k3s/dependencies.go (100%) create mode 100644 pkg/collection/k3s/k3s.go rename pkg/{content => collection}/k3s/k3s_test.go (100%) create mode 100644 pkg/content/file/config.go delete mode 100644 pkg/content/k3s/k3s.go create mode 100644 pkg/layout/artifact.go create mode 100644 pkg/layout/store.go diff --git a/.gitignore b/.gitignore index 5e57de3d..96e1c508 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ dist/ tmp/ bin/ pkg.yaml +store/ \ No newline at end of file diff --git a/.goreleaser.yaml b/.goreleaser.yaml index a85ff430..9fbd4b32 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -4,13 +4,21 @@ builds: goos: - linux - darwin + - windows goarch: - amd64 - arm64 env: - CGO_ENABLED=0 -# flags: -# - -tags=containers_image_openpgp containers_image_ostree -#release: -# extra_files: -# - glob: ./pkg.tar.zst \ No newline at end of file + +universal_binaries: + - replace: true + +brews: + - name: hauler + tap: + owner: rancherfederal + name: homebrew-tap + token: "{{ .Env.HOMEBREW_TAP_GITHUB_TOKEN }}" + folder: Formula + description: "Hauler CLI" diff --git a/cmd/hauler/cli/cli.go b/cmd/hauler/cli/cli.go index 3a09cd24..59d92284 100644 --- a/cmd/hauler/cli/cli.go +++ b/cmd/hauler/cli/cli.go @@ -2,20 +2,21 @@ package cli import ( "context" - "fmt" + "errors" "os" "path/filepath" "github.com/spf13/cobra" + "github.com/rancherfederal/hauler/pkg/cache" "github.com/rancherfederal/hauler/pkg/log" "github.com/rancherfederal/hauler/pkg/store" ) type rootOpts struct { logLevel string - dataDir string cacheDir string + storeDir string } var ro = &rootOpts{} @@ -35,8 +36,8 @@ func New() *cobra.Command { pf := cmd.PersistentFlags() pf.StringVarP(&ro.logLevel, "log-level", "l", "info", "") - pf.StringVar(&ro.dataDir, "content-dir", "", "Location of where to create and store contents (defaults to ~/.local/hauler)") - pf.StringVar(&ro.cacheDir, "cache", "", "Location of where to store cache data (defaults to XDG_CACHE_DIR/hauler)") + pf.StringVar(&ro.cacheDir, "cache", "", "Location of where to store cache data (defaults to $XDG_CACHE_DIR/hauler)") + pf.StringVarP(&ro.storeDir, "store", "s", "", "Location to create store at (defaults to $PWD/store)") // Add subcommands addDownload(cmd) @@ -46,37 +47,56 @@ func New() *cobra.Command { } func (o *rootOpts) getStore(ctx context.Context) (*store.Store, error) { - dir := o.dataDir + lgr := log.FromContext(ctx) + dir := o.storeDir - if o.dataDir == "" { - // Default to userspace - home, err := os.UserHomeDir() + if dir == "" { + lgr.Debugf("no store path specified, defaulting to $PWD/store") + pwd, err := os.Getwd() if err != nil { return nil, err } - abs, _ := filepath.Abs(filepath.Join(home, ".local/hauler/store")) - if err := os.MkdirAll(abs, os.ModePerm); err != nil { + dir = filepath.Join(pwd, "store") + } + + abs, err := filepath.Abs(dir) + if err != nil { + return nil, err + } + + lgr.Debugf("using store at %s", abs) + if _, err := os.Stat(abs); errors.Is(err, os.ErrNotExist) { + err := os.Mkdir(abs, os.ModePerm) + if err != nil { return nil, err } + } else if err != nil { + return nil, err + } - dir = abs - } else { - // Make sure directory exists and we can write to it - if f, err := os.Stat(o.dataDir); err != nil { - return nil, err - } else if !f.IsDir() { - return nil, fmt.Errorf("%s is not a directory", o.dataDir) - } // TODO: Add writeable check + s := store.NewStore(ctx, abs) + return s, nil +} - abs, err := filepath.Abs(o.dataDir) +func (o *rootOpts) getCache(ctx context.Context) (cache.Cache, error) { + dir := o.cacheDir + + if dir == "" { + // Default to $XDG_CACHE_DIR + cachedir, err := os.UserCacheDir() if err != nil { return nil, err } + abs, _ := filepath.Abs(filepath.Join(cachedir, "hauler")) + if err := os.MkdirAll(abs, os.ModePerm); err != nil { + return nil, err + } + dir = abs } - s := store.NewStore(ctx, dir) - return s, nil + c := cache.NewFilesystem(dir) + return c, nil } diff --git a/cmd/hauler/cli/download.go b/cmd/hauler/cli/download.go index 579323fa..8c805c7f 100644 --- a/cmd/hauler/cli/download.go +++ b/cmd/hauler/cli/download.go @@ -10,8 +10,24 @@ func addDownload(parent *cobra.Command) { o := &download.Opts{} cmd := &cobra.Command{ - Use: "download", - Short: "Download OCI content from a registry and populate it on disk", + Use: "download", + Short: "Download OCI content from a registry and populate it on disk", + Long: `Locate OCI content based on it's reference in a compatible registry and download the contents to disk. + +Note that the content type determines it's format on disk. Hauler's built in content types act as follows: + + - File: as a file named after the pushed contents source name (ex: hauler/my-file.yaml:latest --> my-file.yaml) + - Image: as a .tar named after the image (ex: alpine:latest --> alpine:latest.tar) + - Chart: as a .tar.gz named after the chart (ex: grafana/loki:2.0.2 --> grafana-loki-2.0.2.tar.gz)`, + Example: ` +# Download a file +hauler dl hauler/my-file.yaml:latest + +# Download an image +hauler dl rancher/k3s:v1.22.2-k3s2 + +# Download a chart +hauler dl hauler/longhorn:1.2.0`, Aliases: []string{"dl"}, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, arg []string) error { diff --git a/cmd/hauler/cli/download/download.go b/cmd/hauler/cli/download/download.go index 18b377c8..ff383880 100644 --- a/cmd/hauler/cli/download/download.go +++ b/cmd/hauler/cli/download/download.go @@ -2,34 +2,39 @@ package download import ( "context" + "encoding/json" "fmt" + "path" - "github.com/containerd/containerd/images" "github.com/containerd/containerd/remotes/docker" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" "oras.land/oras-go/pkg/content" "oras.land/oras-go/pkg/oras" + "github.com/rancherfederal/hauler/pkg/artifact/types" "github.com/rancherfederal/hauler/pkg/log" ) type Opts struct { DestinationDir string + OutputFile string } func (o *Opts) AddArgs(cmd *cobra.Command) { f := cmd.Flags() f.StringVar(&o.DestinationDir, "dir", "", "Directory to save contents to (defaults to current directory)") + f.StringVarP(&o.OutputFile, "output", "o", "", "(Optional) Override name of file to save.") } func Cmd(ctx context.Context, o *Opts, reference string) error { - l := log.FromContext(ctx) - l.Debugf("running command `hauler download`") + lgr := log.FromContext(ctx) + lgr.Debugf("running command `hauler download`") cs := content.NewFileStore(o.DestinationDir) defer cs.Close() @@ -39,36 +44,86 @@ func Cmd(ctx context.Context, o *Opts, reference string) error { return err } - resolver := docker.NewResolver(docker.ResolverOptions{}) + // resolver := docker.NewResolver(docker.ResolverOptions{}) desc, err := remote.Get(ref) if err != nil { return err } - l.Debugf("Getting content of media type: %s", desc.MediaType) - switch desc.MediaType { - case ocispec.MediaTypeImageManifest: - desc, artifacts, err := oras.Pull(ctx, resolver, ref.Name(), cs, oras.WithPullBaseHandler()) + manifestData, err := desc.RawManifest() + if err != nil { + return err + } + + var manifest ocispec.Manifest + if err := json.Unmarshal(manifestData, &manifest); err != nil { + return err + } + + // TODO: These need to be factored out into each of the contents own logic + switch manifest.Config.MediaType { + case types.DockerConfigJSON, types.OCIManifestSchema1: + lgr.Infof("identified [image] (%s) content", manifest.Config.MediaType) + img, err := remote.Image(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain)) if err != nil { return err } - // TODO: Better logging - _ = desc - _ = artifacts - // l.Infof("Downloaded %d artifacts: %s", len(artifacts), content.ResolveName(desc)) + outputFile := o.OutputFile + if outputFile == "" { + outputFile = fmt.Sprintf("%s:%s.tar", path.Base(ref.Context().RepositoryStr()), ref.Identifier()) + } - case images.MediaTypeDockerSchema2Manifest: - img, err := remote.Image(ref, remote.WithAuthFromKeychain(authn.DefaultKeychain)) + if err := tarball.WriteToFile(outputFile, ref, img); err != nil { + return err + } + + lgr.Infof("downloaded [%s] to [%s]", ref.Name(), outputFile) + + case types.FileMediaType: + lgr.Infof("identified [file] (%s) content", manifest.Config.MediaType) + + fs := content.NewFileStore(o.DestinationDir) + + resolver := docker.NewResolver(docker.ResolverOptions{}) + mdesc, descs, err := oras.Pull(ctx, resolver, ref.Name(), fs) if err != nil { return err } - _ = img + lgr.Infof("downloaded [%d] files with digest [%s]", len(descs), mdesc) + + case types.ChartLayerMediaType, types.ChartConfigMediaType: + lgr.Infof("identified [chart] (%s) content", manifest.Config.MediaType) + + fs := content.NewFileStore(o.DestinationDir) + + resolver := docker.NewResolver(docker.ResolverOptions{}) + mdesc, _, err := oras.Pull(ctx, resolver, ref.Name(), fs) + if err != nil { + return err + } + + lgr.Infof("downloaded chart [%s] with digest [%s]", "donkey", mdesc.Digest.String()) + default: - return fmt.Errorf("unknown media type: %s", desc.MediaType) + return fmt.Errorf("unrecognized content type: %s", manifest.Config.MediaType) } return nil } + +func getManifest(ctx context.Context, ref string) (*remote.Descriptor, error) { + r, err := name.ParseReference(ref) + if err != nil { + return nil, fmt.Errorf("parsing reference %q: %v", ref, err) + } + + desc, err := remote.Get(r, remote.WithContext(ctx)) + if err != nil { + return nil, err + } + + return desc, nil +} diff --git a/cmd/hauler/cli/download/download_test.go b/cmd/hauler/cli/download/download_test.go new file mode 100644 index 00000000..701775cc --- /dev/null +++ b/cmd/hauler/cli/download/download_test.go @@ -0,0 +1,38 @@ +package download + +import ( + "context" + "testing" +) + +func TestCmd(t *testing.T) { + ctx := context.Background() + + type args struct { + ctx context.Context + o *Opts + reference string + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "should work", + args: args{ + ctx: ctx, + o: &Opts{DestinationDir: ""}, + reference: "localhost:3000/hauler/file.txt:latest", + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := Cmd(tt.args.ctx, tt.args.o, tt.args.reference); (err != nil) != tt.wantErr { + t.Errorf("Cmd() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/cmd/hauler/cli/store.go b/cmd/hauler/cli/store.go index 4bd0d885..ee2e9d67 100644 --- a/cmd/hauler/cli/store.go +++ b/cmd/hauler/cli/store.go @@ -21,8 +21,10 @@ func addStore(parent *cobra.Command) { addStoreLoad(), addStoreSave(), addStoreServe(), + addStoreList(), + addStoreCopy(), - // TODO: Remove this in favor of sync only + // TODO: Remove this in favor of sync? addStoreAdd(), ) @@ -67,7 +69,12 @@ func addStoreSync() *cobra.Command { return err } - return store.SyncCmd(ctx, o, s) + c, err := ro.getCache(ctx) + if err != nil { + return err + } + + return store.SyncCmd(ctx, o, s, c) }, } o.AddFlags(cmd) @@ -143,6 +150,52 @@ func addStoreSave() *cobra.Command { return cmd } +func addStoreList() *cobra.Command { + o := &store.ListOpts{} + + cmd := &cobra.Command{ + Use: "list", + Short: "List all content references in a store", + Args: cobra.ExactArgs(0), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + s, err := ro.getStore(ctx) + if err != nil { + return err + } + + return store.ListCmd(ctx, o, s) + }, + } + o.AddFlags(cmd) + + return cmd +} + +func addStoreCopy() *cobra.Command { + o := &store.CopyOpts{} + + cmd := &cobra.Command{ + Use: "copy", + Short: "Copy all store contents to another OCI registry", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + + s, err := ro.getStore(ctx) + if err != nil { + return err + } + + return store.CopyCmd(ctx, o, s, args[0]) + }, + } + o.AddFlags(cmd) + + return cmd +} + func addStoreAdd() *cobra.Command { cmd := &cobra.Command{ Use: "add", @@ -176,9 +229,12 @@ func addStoreAddFile() *cobra.Command { return err } - ref := args[0] + c, err := ro.getCache(ctx) + if err != nil { + return err + } - return store.AddFileCmd(ctx, o, s, ref) + return store.AddFileCmd(ctx, o, s, c, args[0]) }, } o.AddFlags(cmd) @@ -201,9 +257,12 @@ func addStoreAddImage() *cobra.Command { return err } - ref := args[0] + c, err := ro.getCache(ctx) + if err != nil { + return err + } - return store.AddImageCmd(ctx, o, s, ref) + return store.AddImageCmd(ctx, o, s, c, args[0]) }, } o.AddFlags(cmd) @@ -233,7 +292,12 @@ hauler store add chart rancher --repo "https://releases.rancher.com/server-chart return err } - return store.AddChartCmd(ctx, o, s, args[0]) + c, err := ro.getCache(ctx) + if err != nil { + return err + } + + return store.AddChartCmd(ctx, o, s, c, args[0]) }, } o.AddFlags(cmd) diff --git a/cmd/hauler/cli/store/add.go b/cmd/hauler/cli/store/add.go index fb3aac53..caa6eff7 100644 --- a/cmd/hauler/cli/store/add.go +++ b/cmd/hauler/cli/store/add.go @@ -2,10 +2,13 @@ package store import ( "context" + "path/filepath" + "github.com/google/go-containerregistry/pkg/name" "github.com/spf13/cobra" "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" + "github.com/rancherfederal/hauler/pkg/cache" "github.com/rancherfederal/hauler/pkg/content/chart" "github.com/rancherfederal/hauler/pkg/content/file" "github.com/rancherfederal/hauler/pkg/content/image" @@ -22,20 +25,52 @@ func (o *AddFileOpts) AddFlags(cmd *cobra.Command) { f.StringVarP(&o.Name, "name", "n", "", "(Optional) Name to assign to file in store") } -func AddFileCmd(ctx context.Context, o *AddFileOpts, s *store.Store, ref string) error { - l := log.FromContext(ctx) - l.Debugf("running cli command `hauler store add`") +func AddFileCmd(ctx context.Context, o *AddFileOpts, s *store.Store, c cache.Cache, reference string) error { + lgr := log.FromContext(ctx) + lgr.Debugf("running cli command `hauler store add`") s.Open() defer s.Close() cfg := v1alpha1.File{ - Ref: ref, + Ref: reference, Name: o.Name, } - f := file.NewFile(cfg) - return s.Add(ctx, f) + return storeFile(ctx, s, c, cfg) +} + +func storeFile(ctx context.Context, s *store.Store, c cache.Cache, fi v1alpha1.File) error { + lgr := log.FromContext(ctx) + + if fi.Name == "" { + base := filepath.Base(fi.Ref) + fi.Name = filepath.Base(fi.Ref) + lgr.Warnf("no name specified for file reference [%s], using base filepath: [%s]", fi.Ref, base) + } + + f, err := file.NewFile(fi.Ref, fi.Name) + if err != nil { + return err + } + + ref, err := name.ParseReference(fi.Name) + if err != nil { + return err + } + + if c != nil { + cf := cache.Oci(f, c) + f = cf + } + + desc, err := s.Add(ctx, f, ref) + if err != nil { + return err + } + + lgr.Infof("added file [%s] to store at [%s] with manifest digest [%s]", fi.Ref, ref.Context().RepositoryStr(), desc.Digest.String()) + return nil } type AddImageOpts struct { @@ -47,23 +82,48 @@ func (o *AddImageOpts) AddFlags(cmd *cobra.Command) { _ = f } -func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Store, ref string) error { - l := log.FromContext(ctx) - l.Debugf("running cli command `hauler store add image`") +func AddImageCmd(ctx context.Context, o *AddImageOpts, s *store.Store, c cache.Cache, reference string) error { + lgr := log.FromContext(ctx) + lgr.Debugf("running cli command `hauler store add image`") s.Open() defer s.Close() cfg := v1alpha1.Image{ - Ref: ref, + Ref: reference, } - i := image.NewImage(cfg) - return s.Add(ctx, i) + return storeImage(ctx, s, c, cfg) +} + +func storeImage(ctx context.Context, s *store.Store, c cache.Cache, i v1alpha1.Image) error { + lgr := log.FromContext(ctx) + + img, err := image.NewImage(i.Ref) + if err != nil { + return err + } + + ref, err := name.ParseReference(i.Ref) + if err != nil { + return err + } + + if c != nil { + ci := cache.Oci(img, c) + img = ci + } + + desc, err := s.Add(ctx, img, ref) + if err != nil { + return err + } + + lgr.Infof("added image [%s] to store at [%s] with manifest digest [%s]", i.Ref, ref.Context().RepositoryStr(), desc.Digest.String()) + return nil } type AddChartOpts struct { - Name string Version string RepoURL string @@ -86,19 +146,50 @@ func (o *AddChartOpts) AddFlags(cmd *cobra.Command) { f.StringVar(&o.Version, "version", "", "(Optional) Version of the chart to download, defaults to latest if not specified") } -func AddChartCmd(ctx context.Context, o *AddChartOpts, s *store.Store, name string) error { - l := log.FromContext(ctx) - l.Debugf("running cli command `hauler store add chart`") +func AddChartCmd(ctx context.Context, o *AddChartOpts, s *store.Store, c cache.Cache, chartName string) error { + lgr := log.FromContext(ctx) + lgr.Debugf("running cli command `hauler store add chart`") s.Open() defer s.Close() cfg := v1alpha1.Chart{ - Name: name, + Name: chartName, RepoURL: o.RepoURL, Version: o.Version, } - c := chart.NewChart(cfg) - return s.Add(ctx, c) + return storeChart(ctx, s, c, cfg) +} + +func storeChart(ctx context.Context, s *store.Store, c cache.Cache, ch v1alpha1.Chart) error { + lgr := log.FromContext(ctx) + + chrt, err := chart.NewChart(ch.Name, ch.RepoURL, ch.Version) + if err != nil { + return err + } + + tag := ch.Version + if tag == "" { + tag = name.DefaultTag + } + + ref, err := name.ParseReference(ch.Name, name.WithDefaultTag(tag)) + if err != nil { + return err + } + + if c != nil { + cch := cache.Oci(chrt, c) + chrt = cch + } + + desc, err := s.Add(ctx, chrt, ref) + if err != nil { + return err + } + + lgr.Infof("added chart [%s] to store at [%s] with manifest digest [%s]", ch.Name, ref.Context().RepositoryStr(), desc.Digest.String()) + return nil } diff --git a/cmd/hauler/cli/store/sync.go b/cmd/hauler/cli/store/sync.go index db9e597b..546d6eec 100644 --- a/cmd/hauler/cli/store/sync.go +++ b/cmd/hauler/cli/store/sync.go @@ -10,11 +10,8 @@ import ( "k8s.io/apimachinery/pkg/util/yaml" "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" + "github.com/rancherfederal/hauler/pkg/cache" "github.com/rancherfederal/hauler/pkg/content" - "github.com/rancherfederal/hauler/pkg/content/chart" - "github.com/rancherfederal/hauler/pkg/content/file" - "github.com/rancherfederal/hauler/pkg/content/image" - "github.com/rancherfederal/hauler/pkg/content/k3s" "github.com/rancherfederal/hauler/pkg/log" "github.com/rancherfederal/hauler/pkg/store" ) @@ -29,15 +26,21 @@ func (o *SyncOpts) AddFlags(cmd *cobra.Command) { f.StringSliceVarP(&o.ContentFiles, "files", "f", []string{}, "Path to content files") } -func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store) error { +func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store, c cache.Cache) error { l := log.FromContext(ctx) l.Debugf("running cli command `hauler store sync`") + // Start from an empty store (contents are cached elsewhere) + l.Debugf("flushing any existing content in store: %s", s.DataDir) + if err := s.Flush(ctx); err != nil { + return err + } + s.Open() defer s.Close() for _, filename := range o.ContentFiles { - l.Debugf("Syncing content file: '%s'", filename) + l.Debugf("processing content file: '%s'", filename) fi, err := os.Open(filename) if err != nil { return err @@ -64,7 +67,7 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store) error { return err } - l.Infof("Syncing content from: '%s'", gvk.String()) + l.Infof("syncing [%s/%s] to [%s]", gvk.APIVersion, gvk.Kind, s.DataDir) switch gvk.Kind { case v1alpha1.FilesContentKind: @@ -74,8 +77,8 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store) error { } for _, f := range cfg.Spec.Files { - oci := file.NewFile(f) - if err := s.Add(ctx, oci); err != nil { + err := storeFile(ctx, s, c, f) + if err != nil { return err } } @@ -87,9 +90,8 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store) error { } for _, i := range cfg.Spec.Images { - oci := image.NewImage(i) - - if err := s.Add(ctx, oci); err != nil { + err := storeImage(ctx, s, c, i) + if err != nil { return err } } @@ -100,27 +102,12 @@ func SyncCmd(ctx context.Context, o *SyncOpts, s *store.Store) error { return err } - for _, c := range cfg.Spec.Charts { - oci := chart.NewChart(c) - if err := s.Add(ctx, oci); err != nil { + for _, ch := range cfg.Spec.Charts { + err := storeChart(ctx, s, c, ch) + if err != nil { return err } } - - case v1alpha1.DriverContentKind: - var cfg v1alpha1.Driver - if err := yaml.Unmarshal(doc, &cfg); err != nil { - return err - } - - oci, err := k3s.NewK3s(cfg.Spec.Version) - if err != nil { - return err - } - - if err := s.Add(ctx, oci); err != nil { - return err - } } } } diff --git a/go.mod b/go.mod index 7ef7ac9a..6d381d60 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,6 @@ require ( github.com/mattn/go-isatty v0.0.14 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/mholt/archiver/v3 v3.5.0 - github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6 github.com/rancher/fleet v0.3.6 github.com/rancher/fleet/pkg/apis v0.0.0 @@ -23,7 +22,6 @@ require ( github.com/spf13/cobra v1.2.1 github.com/ulikunitz/xz v0.5.10 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b // indirect - go.etcd.io/bbolt v1.3.6 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect @@ -33,7 +31,6 @@ require ( k8s.io/cli-runtime v0.22.1 k8s.io/client-go v11.0.1-0.20190816222228-6d55c1b1f1ca+incompatible oras.land/oras-go v0.4.0 - sigs.k8s.io/cli-utils v0.23.1 sigs.k8s.io/controller-runtime v0.9.0 ) @@ -173,6 +170,7 @@ require ( github.com/morikuni/aec v1.0.0 // indirect github.com/mozilla-services/yaml v0.0.0-20191106225358-5c216288813c // indirect github.com/nwaples/rardecode v1.1.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pierrec/lz4 v2.0.5+incompatible // indirect github.com/pierrec/lz4/v4 v4.0.3 // indirect @@ -240,6 +238,7 @@ require ( k8s.io/kubectl v0.22.1 // indirect k8s.io/kubernetes v1.21.3 // indirect k8s.io/utils v0.0.0-20210527160623-6fdb442a123b // indirect + sigs.k8s.io/cli-utils v0.23.1 // indirect sigs.k8s.io/kustomize/api v0.8.8 // indirect sigs.k8s.io/kustomize/kyaml v0.10.17 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect diff --git a/go.sum b/go.sum index b4cf4a05..78275709 100644 --- a/go.sum +++ b/go.sum @@ -626,7 +626,6 @@ github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= -github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM= github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= @@ -1683,8 +1682,6 @@ github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wK go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= @@ -1717,18 +1714,15 @@ go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0H go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= gocloud.dev v0.19.0/go.mod h1:SmKwiR8YwIMMJvQBKLsC3fHNyMwXLw3PMDO+VVteJMI= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -2002,7 +1996,6 @@ golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201110211018-35f3e6cf4a65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/artifact/config.go b/pkg/artifact/config.go new file mode 100644 index 00000000..7b5155b9 --- /dev/null +++ b/pkg/artifact/config.go @@ -0,0 +1,10 @@ +package artifact + +import v1 "github.com/google/go-containerregistry/pkg/v1" + +type Config interface { + // Raw returns the config bytes + Raw() ([]byte, error) + + Descriptor() (v1.Descriptor, error) +} diff --git a/pkg/artifact/local/layer.go b/pkg/artifact/local/layer.go new file mode 100644 index 00000000..e25f24d5 --- /dev/null +++ b/pkg/artifact/local/layer.go @@ -0,0 +1,127 @@ +package local + +import ( + "io" + + v1 "github.com/google/go-containerregistry/pkg/v1" + gtypes "github.com/google/go-containerregistry/pkg/v1/types" + + "github.com/rancherfederal/hauler/pkg/artifact/types" +) + +type Opener func() (io.ReadCloser, error) + +func LayerFromOpener(opener Opener, opts ...LayerOption) (v1.Layer, error) { + var err error + + layer := &layer{ + mediaType: types.UnknownLayer, + annotations: make(map[string]string, 1), + } + + layer.uncompressedOpener = opener + layer.compressedOpener = func() (io.ReadCloser, error) { + rc, err := opener() + if err != nil { + return nil, err + } + // TODO: actually compress this + return rc, nil + } + + for _, opt := range opts { + opt(layer) + } + + if layer.digest, layer.size, err = compute(layer.uncompressedOpener); err != nil { + return nil, err + } + + if layer.diffID, _, err = compute(layer.compressedOpener); err != nil { + return nil, err + } + + return layer, nil +} + +func compute(opener Opener) (v1.Hash, int64, error) { + rc, err := opener() + if err != nil { + return v1.Hash{}, 0, err + } + defer rc.Close() + return v1.SHA256(rc) +} + +type LayerOption func(*layer) + +func WithMediaType(mt string) LayerOption { + return func(l *layer) { + l.mediaType = mt + } +} + +func WithAnnotations(annotations map[string]string) LayerOption { + return func(l *layer) { + if l.annotations == nil { + l.annotations = make(map[string]string) + } + l.annotations = annotations + } +} + +type layer struct { + digest v1.Hash + diffID v1.Hash + size int64 + compressedOpener Opener + uncompressedOpener Opener + mediaType string + annotations map[string]string + urls []string +} + +func (l layer) Descriptor() (*v1.Descriptor, error) { + digest, err := l.Digest() + if err != nil { + return nil, err + } + mt, err := l.MediaType() + if err != nil { + return nil, err + } + return &v1.Descriptor{ + MediaType: mt, + Size: l.size, + Digest: digest, + Annotations: l.annotations, + URLs: l.urls, + + // TODO: Allow platforms + Platform: nil, + }, nil +} + +func (l layer) Digest() (v1.Hash, error) { + return l.digest, nil +} + +func (l layer) DiffID() (v1.Hash, error) { + return l.diffID, nil +} + +func (l layer) Compressed() (io.ReadCloser, error) { + return l.compressedOpener() +} + +func (l layer) Uncompressed() (io.ReadCloser, error) { + return l.uncompressedOpener() +} + +func (l layer) Size() (int64, error) { + return l.size, nil +} + +func (l layer) MediaType() (gtypes.MediaType, error) { + return gtypes.MediaType(l.mediaType), nil +} diff --git a/pkg/artifact/oci.go b/pkg/artifact/oci.go new file mode 100644 index 00000000..7ee2d16b --- /dev/null +++ b/pkg/artifact/oci.go @@ -0,0 +1,20 @@ +package artifact + +import ( + "github.com/google/go-containerregistry/pkg/v1" +) + +// OCI is the bare minimum we need to represent an artifact in an OCI layout +// Oci is a general form of v1.Image that conforms to the OCI artifacts-spec instead of the images-spec +// At a high level, it is not constrained by an Image's config, manifests, and layer ordinality +// This specific implementation fully encapsulates v1.Layer's within a more generic form +type OCI interface { + MediaType() string + + // ManifestData() ([]byte, error) + Manifest() (*v1.Manifest, error) + + RawConfig() ([]byte, error) + + Layers() ([]v1.Layer, error) +} diff --git a/pkg/artifact/types/types.go b/pkg/artifact/types/types.go new file mode 100644 index 00000000..10d5508d --- /dev/null +++ b/pkg/artifact/types/types.go @@ -0,0 +1,26 @@ +package types + +const ( + OCIManifestSchema1 = "application/vnd.oci.image.manifest.v1+json" + DockerManifestSchema2 = "application/vnd.docker.distribution.manifest.v2+json" + DockerConfigJSON = "application/vnd.docker.container.image.v1+json" + UnknownManifest = "application/vnd.hauler.cattle.io.unknown.v1+json" + + UnknownLayer = "application/vnd.content.hauler.unknown.layer" + FileLayerMediaType = "application/vnd.content.hauler.file.layer.v1" + FileMediaType = "application/vnd.content.hauler.file.config.v1+json" + + // ConfigMediaType is the reserved media type for the Helm chart manifest config + ChartConfigMediaType = "application/vnd.cncf.helm.config.v1+json" + + // ChartLayerMediaType is the reserved media type for Helm chart package content + ChartLayerMediaType = "application/vnd.cncf.helm.chart.content.v1.tar+gzip" + + // ProvLayerMediaType is the reserved media type for Helm chart provenance files + ProvLayerMediaType = "application/vnd.cncf.helm.chart.provenance.v1.prov" + + OCIVendorPrefix = "vnd.oci" + DockerVendorPrefix = "vnd.docker" + HaulerVendorPrefix = "vnd.hauler" + OCIImageIndexFile = "index.json" +) diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go new file mode 100644 index 00000000..781bcd92 --- /dev/null +++ b/pkg/cache/cache.go @@ -0,0 +1,102 @@ +package cache + +import ( + "errors" + "io" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" + + "github.com/rancherfederal/hauler/pkg/artifact" +) + +type Cache interface { + Put(v1.Layer) (v1.Layer, error) + + Get(v1.Hash) (v1.Layer, error) +} + +var ErrLayerNotFound = errors.New("layer not found") + +type oci struct { + artifact.OCI + + c Cache +} + +func Oci(o artifact.OCI, c Cache) artifact.OCI { + return &oci{ + OCI: o, + c: c, + } +} + +func (o *oci) Layers() ([]v1.Layer, error) { + ls, err := o.OCI.Layers() + if err != nil { + return nil, err + } + + var out []v1.Layer + for _, l := range ls { + out = append(out, &lazyLayer{inner: l, c: o.c}) + } + return out, nil +} + +type lazyLayer struct { + inner v1.Layer + c Cache +} + +func (l *lazyLayer) Compressed() (io.ReadCloser, error) { + digest, err := l.inner.Digest() + if err != nil { + return nil, err + } + + layer, err := l.getOrPut(digest) + if err != nil { + return nil, err + } + + return layer.Compressed() +} + +func (l *lazyLayer) Uncompressed() (io.ReadCloser, error) { + diffID, err := l.inner.DiffID() + if err != nil { + return nil, err + } + + layer, err := l.getOrPut(diffID) + if err != nil { + return nil, err + } + + return layer.Uncompressed() +} + +func (l *lazyLayer) getOrPut(h v1.Hash) (v1.Layer, error) { + var layer v1.Layer + if cl, err := l.c.Get(h); err == nil { + layer = cl + + } else if err == ErrLayerNotFound { + rl, err := l.c.Put(l.inner) + if err != nil { + return nil, err + } + layer = rl + + } else { + return nil, err + } + + return layer, nil +} + +func (l *lazyLayer) Size() (int64, error) { return l.inner.Size() } +func (l *lazyLayer) DiffID() (v1.Hash, error) { return l.inner.Digest() } +func (l *lazyLayer) Digest() (v1.Hash, error) { return l.inner.Digest() } +func (l *lazyLayer) MediaType() (types.MediaType, error) { return l.inner.MediaType() } diff --git a/pkg/cache/doc.go b/pkg/cache/doc.go new file mode 100644 index 00000000..9a0a2b8a --- /dev/null +++ b/pkg/cache/doc.go @@ -0,0 +1,5 @@ +package cache + +/* +This package is _heavily_ influenced by go-containerregistry and it's cache implementation: https://github.com/google/go-containerregistry/tree/main/pkg/v1/cache +*/ diff --git a/pkg/cache/filesystem.go b/pkg/cache/filesystem.go new file mode 100644 index 00000000..fd3c547d --- /dev/null +++ b/pkg/cache/filesystem.go @@ -0,0 +1,120 @@ +package cache + +import ( + "io" + "os" + "path/filepath" + + v1 "github.com/google/go-containerregistry/pkg/v1" + + "github.com/rancherfederal/hauler/pkg/artifact/local" +) + +type fs struct { + root string +} + +func NewFilesystem(root string) Cache { + return &fs{root: root} +} + +func (f *fs) Put(l v1.Layer) (v1.Layer, error) { + digest, err := l.Digest() + if err != nil { + return nil, err + } + diffID, err := l.DiffID() + if err != nil { + return nil, err + } + return &cachedLayer{ + Layer: l, + root: f.root, + digest: digest, + diffID: diffID, + }, nil +} + +func (f *fs) Get(h v1.Hash) (v1.Layer, error) { + opener := f.open(h) + l, err := local.LayerFromOpener(opener) + if os.IsNotExist(err) { + return nil, ErrLayerNotFound + } + return l, err +} + +func (f *fs) open(h v1.Hash) local.Opener { + return func() (io.ReadCloser, error) { + return os.Open(layerpath(f.root, h)) + } +} + +type cachedLayer struct { + v1.Layer + + root string + digest, diffID v1.Hash +} + +func (l *cachedLayer) create(h v1.Hash) (io.WriteCloser, error) { + lp := layerpath(l.root, h) + if err := os.MkdirAll(filepath.Dir(lp), os.ModePerm); err != nil { + return nil, err + } + return os.Create(lp) +} + +func (l *cachedLayer) Compressed() (io.ReadCloser, error) { + f, err := l.create(l.digest) + if err != nil { + return nil, nil + } + rc, err := l.Layer.Compressed() + if err != nil { + return nil, err + } + return &readcloser{ + t: io.TeeReader(rc, f), + closes: []func() error{rc.Close, f.Close}, + }, nil +} + +func (l *cachedLayer) Uncompressed() (io.ReadCloser, error) { + f, err := l.create(l.diffID) + if err != nil { + return nil, err + } + rc, err := l.Layer.Uncompressed() + if err != nil { + return nil, err + } + return &readcloser{ + t: io.TeeReader(rc, f), + closes: []func() error{rc.Close, f.Close}, + }, nil +} + +func layerpath(root string, h v1.Hash) string { + return filepath.Join(root, h.Algorithm, h.Hex) +} + +type readcloser struct { + t io.Reader + closes []func() error +} + +func (rc *readcloser) Read(b []byte) (int, error) { + return rc.t.Read(b) +} + +func (rc *readcloser) Close() error { + var err error + for _, c := range rc.closes { + lastErr := c() + if err == nil { + err = lastErr + } + } + return err +} diff --git a/pkg/content/k3s/dependencies.go b/pkg/collection/k3s/dependencies.go similarity index 100% rename from pkg/content/k3s/dependencies.go rename to pkg/collection/k3s/dependencies.go diff --git a/pkg/collection/k3s/k3s.go b/pkg/collection/k3s/k3s.go new file mode 100644 index 00000000..717454aa --- /dev/null +++ b/pkg/collection/k3s/k3s.go @@ -0,0 +1,46 @@ +package k3s + +// type K3s struct { +// Files []file.File +// Images []image.Image +// } +// +// func NewK3s(version string) (*K3s, error) { +// bom, err := newDependencies("k3s", version) +// if err != nil { +// return nil, err +// } +// +// var files []file.File +// for _, f := range bom.files.Spec.Files { +// fi := file.NewFile(f) +// files = append(files, fi) +// } +// +// var images []image.Image +// for _, i := range bom.images.Spec.Images { +// img := image.NewImage(i) +// images = append(images, img) +// } +// +// return &K3s{ +// Files: files, +// Images: images, +// }, nil +// } +// +// func (k *K3s) Copy(ctx context.Context, registry string) error { +// for _, f := range k.Files { +// if err := f.Copy(ctx, registry); err != nil { +// return err +// } +// } +// +// for _, i := range k.Images { +// if err := i.Copy(ctx, registry); err != nil { +// return err +// } +// } +// +// return nil +// } diff --git a/pkg/content/k3s/k3s_test.go b/pkg/collection/k3s/k3s_test.go similarity index 100% rename from pkg/content/k3s/k3s_test.go rename to pkg/collection/k3s/k3s_test.go diff --git a/pkg/content/chart/chart.go b/pkg/content/chart/chart.go index 31ac7942..3874c925 100644 --- a/pkg/content/chart/chart.go +++ b/pkg/content/chart/chart.go @@ -2,139 +2,135 @@ package chart import ( "bytes" - "context" "encoding/json" - "fmt" + "io" "os" - "strings" + "path/filepath" - "github.com/containerd/containerd/remotes" - "github.com/containerd/containerd/remotes/docker" - "github.com/google/go-containerregistry/pkg/name" + gv1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + gtypes "github.com/google/go-containerregistry/pkg/v1/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/cli" - "k8s.io/client-go/util/jsonpath" - "oras.land/oras-go/pkg/content" - "oras.land/oras-go/pkg/oras" - "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" - "github.com/rancherfederal/hauler/pkg/log" + "github.com/rancherfederal/hauler/pkg/artifact" + "github.com/rancherfederal/hauler/pkg/artifact/local" + "github.com/rancherfederal/hauler/pkg/artifact/types" ) const ( - // OCIScheme is the URL scheme for OCI-based requests - OCIScheme = "oci" - - // CredentialsFileBasename is the filename for auth credentials file - CredentialsFileBasename = "config.json" - - // ConfigMediaType is the reserved media type for the Helm chart manifest config - ConfigMediaType = "application/vnd.cncf.helm.config.v1+json" - // ChartLayerMediaType is the reserved media type for Helm chart package content ChartLayerMediaType = "application/vnd.cncf.helm.chart.content.v1.tar+gzip" - - // ProvLayerMediaType is the reserved media type for Helm chart provenance files - ProvLayerMediaType = "application/vnd.cncf.helm.chart.provenance.v1.prov" ) -type Chart struct { - cfg v1alpha1.Chart - - resolver remotes.Resolver -} +var _ artifact.OCI = (*chrt)(nil) -var defaultKnownImagePaths = []string{ - // Deployments & DaemonSets - "{.spec.template.spec.initContainers[*].image}", - "{.spec.template.spec.containers[*].image}", +type chrt struct { + path string - // Pods - "{.spec.initContainers[*].image}", - "{.spec.containers[*].image}", + annotations map[string]string } -func NewChart(cfg v1alpha1.Chart) Chart { - return Chart{ - cfg: cfg, +func NewChart(name, repo, version string) (artifact.OCI, error) { + cpo := action.ChartPathOptions{ + RepoURL: repo, + Version: version, + } - // TODO: - resolver: docker.NewResolver(docker.ResolverOptions{}), + cp, err := cpo.LocateChart(name, cli.New()) + if err != nil { + return nil, err } + + return &chrt{ + path: cp, + }, nil } -func (c Chart) Copy(ctx context.Context, registry string) error { - var ( - s = content.NewMemoryStore() - l = log.FromContext(ctx) - contentDescriptors []ocispec.Descriptor - ) +func (h *chrt) MediaType() string { + return types.OCIManifestSchema1 +} - chartdata, err := fetch(c.cfg.Name, c.cfg.RepoURL, c.cfg.Version) +func (h *chrt) Manifest() (*gv1.Manifest, error) { + cfgDesc, err := h.configDescriptor() if err != nil { - return err + return nil, err } - ch, err := loader.LoadArchive(bytes.NewBuffer(chartdata)) - if err != nil { - return err + var layerDescs []gv1.Descriptor + ls, err := h.Layers() + for _, l := range ls { + desc, err := partial.Descriptor(l) + if err != nil { + return nil, err + } + layerDescs = append(layerDescs, *desc) } - chartDescriptor := s.Add("", ChartLayerMediaType, chartdata) - contentDescriptors = append(contentDescriptors, chartDescriptor) + return &gv1.Manifest{ + SchemaVersion: 2, + MediaType: gtypes.MediaType(h.MediaType()), + Config: cfgDesc, + Layers: layerDescs, + Annotations: h.annotations, + }, nil +} - configData, _ := json.Marshal(ch.Metadata) - configDescriptor := s.Add("", ConfigMediaType, configData) +func (h *chrt) RawConfig() ([]byte, error) { + ch, err := loader.Load(h.path) + if err != nil { + return nil, err + } + return json.Marshal(ch.Metadata) +} - // TODO: Clean this up - ref, err := name.ParseReference(fmt.Sprintf("hauler/%s:%s", c.cfg.Name, c.cfg.Version), name.WithDefaultRegistry(registry)) +func (h *chrt) configDescriptor() (gv1.Descriptor, error) { + data, err := h.RawConfig() if err != nil { - return err + return gv1.Descriptor{}, err } - l.Infof("Copying chart to: '%s'", ref.Name()) - pushedDesc, err := oras.Push(ctx, c.resolver, ref.Name(), s, contentDescriptors, - oras.WithConfig(configDescriptor), oras.WithNameValidation(nil)) + hash, size, err := gv1.SHA256(bytes.NewBuffer(data)) if err != nil { - return err + return gv1.Descriptor{}, err } - l.Debugf("Copied with descriptor: '%s'", pushedDesc.Digest.String()) - return nil + return gv1.Descriptor{ + MediaType: types.ChartConfigMediaType, + Size: size, + Digest: hash, + }, nil } -func fetch(name, repo, version string) ([]byte, error) { - cpo := action.ChartPathOptions{ - RepoURL: repo, - Version: version, - } - - cp, err := cpo.LocateChart(name, cli.New()) +func (h *chrt) Layers() ([]gv1.Layer, error) { + chartDataLayer, err := h.chartDataLayer() if err != nil { return nil, err } - data, err := os.ReadFile(cp) - if err != nil { - return nil, err - } + return []gv1.Layer{ + chartDataLayer, + // TODO: Add provenance + }, nil +} - return data, nil +func (h *chrt) RawChartData() ([]byte, error) { + return os.ReadFile(h.path) } -func parseJSONPath(data interface{}, parser *jsonpath.JSONPath, template string) ([]string, error) { - buf := new(bytes.Buffer) - if err := parser.Parse(template); err != nil { - return nil, err - } +func (h *chrt) chartDataLayer() (gv1.Layer, error) { + annotations := make(map[string]string) + annotations[ocispec.AnnotationTitle] = filepath.Base(h.path) - if err := parser.Execute(buf, data); err != nil { - return nil, err - } + return local.LayerFromOpener(chartOpener(h.path), + local.WithMediaType(types.ChartLayerMediaType), + local.WithAnnotations(annotations)) +} - f := func(s rune) bool { return s == ' ' } - r := strings.FieldsFunc(buf.String(), f) - return r, nil +func chartOpener(path string) local.Opener { + return func() (io.ReadCloser, error) { + return os.Open(path) + } } diff --git a/pkg/content/chart/chart_test.go b/pkg/content/chart/chart_test.go index 9c796e0d..3b304e54 100644 --- a/pkg/content/chart/chart_test.go +++ b/pkg/content/chart/chart_test.go @@ -1,11 +1,15 @@ -package chart +package chart_test import ( "context" "os" + "path" "testing" + "github.com/google/go-containerregistry/pkg/name" + "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" + "github.com/rancherfederal/hauler/pkg/content/chart" "github.com/rancherfederal/hauler/pkg/log" "github.com/rancherfederal/hauler/pkg/store" ) @@ -37,24 +41,31 @@ func TestChart_Copy(t *testing.T) { }{ // TODO: This test isn't self-contained { - name: "should work", + name: "should work with unversioned chart", cfg: v1alpha1.Chart{ - Name: "rancher", - RepoURL: "https://releases.rancher.com/server-charts/latest", - Version: "2.6.2", + Name: "loki", + RepoURL: "https://grafana.github.io/helm-charts", }, args: args{ ctx: ctx, - registry: s.RegistryURL(), + registry: s.Registry(), }, wantErr: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := NewChart(tt.cfg) - if err := c.Copy(tt.args.ctx, tt.args.registry); (err != nil) != tt.wantErr { - t.Errorf("Copy() error = %v, wantErr %v", err, tt.wantErr) + c, err := chart.NewChart(tt.cfg.Name, tt.cfg.RepoURL, tt.cfg.Version) + if err != nil { + t.Fatal(err) + } + ref, err := name.ParseReference(path.Join("hauler", tt.cfg.Name)) + if err != nil { + t.Fatal(err) + } + + if _, err := s.Add(ctx, c, ref); (err != nil) != tt.wantErr { + t.Error(err) } }) } diff --git a/pkg/content/chart/dependents.go b/pkg/content/chart/dependents.go index aa677eb0..c7540f18 100644 --- a/pkg/content/chart/dependents.go +++ b/pkg/content/chart/dependents.go @@ -4,10 +4,11 @@ import ( "bytes" "encoding/json" "io" + "strings" "github.com/rancher/wrangler/pkg/yaml" "helm.sh/helm/v3/pkg/action" - "helm.sh/helm/v3/pkg/chart" + helmchart "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chartutil" "helm.sh/helm/v3/pkg/kube/fake" "helm.sh/helm/v3/pkg/storage" @@ -19,8 +20,18 @@ import ( "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" ) +var defaultKnownImagePaths = []string{ + // Deployments & DaemonSets + "{.spec.template.spec.initContainers[*].image}", + "{.spec.template.spec.containers[*].image}", + + // Pods + "{.spec.initContainers[*].image}", + "{.spec.containers[*].image}", +} + // ImagesInChart will render a chart and identify all dependent images from it -func ImagesInChart(c *chart.Chart) (v1alpha1.Images, error) { +func ImagesInChart(c *helmchart.Chart) (v1alpha1.Images, error) { objs, err := template(c) if err != nil { return v1alpha1.Images{}, err @@ -64,7 +75,7 @@ func ImagesInChart(c *chart.Chart) (v1alpha1.Images, error) { return ims, nil } -func template(c *chart.Chart) ([]runtime.Object, error) { +func template(c *helmchart.Chart) ([]runtime.Object, error) { s := storage.Init(driver.NewMemory()) templateCfg := &action.Configuration{ @@ -93,3 +104,18 @@ func template(c *chart.Chart) ([]runtime.Object, error) { return yaml.ToObjects(bytes.NewBufferString(release.Manifest)) } + +func parseJSONPath(data interface{}, parser *jsonpath.JSONPath, template string) ([]string, error) { + buf := new(bytes.Buffer) + if err := parser.Parse(template); err != nil { + return nil, err + } + + if err := parser.Execute(buf, data); err != nil { + return nil, err + } + + f := func(s rune) bool { return s == ' ' } + r := strings.FieldsFunc(buf.String(), f) + return r, nil +} diff --git a/pkg/content/content.go b/pkg/content/content.go index d58d5713..b9aadb8b 100644 --- a/pkg/content/content.go +++ b/pkg/content/content.go @@ -1,7 +1,6 @@ package content import ( - "context" "fmt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -10,10 +9,9 @@ import ( "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" ) -type Oci interface { - // Copy relocates content to an OCI compliant registry given a name.Reference - Copy(ctx context.Context, registry string) error -} +const ( + UnknownLayerMediaType = "application/vnd.hauler.cattle.io.unknown" +) func ValidateType(data []byte) (metav1.TypeMeta, error) { var tm metav1.TypeMeta @@ -27,90 +25,3 @@ func ValidateType(data []byte) (metav1.TypeMeta, error) { return tm, nil } - -// // NewFromBytes returns a new Oci object from content bytes -// func NewFromBytes(data []byte) (Oci, error) { -// var tm metav1.TypeMeta -// if err := yaml.Unmarshal(data, &tm); err != nil { -// return nil, err -// } -// -// if tm.GroupVersionKind().GroupVersion() != v1alpha1.GroupVersion { -// return nil, fmt.Errorf("%s is not an understood content type", tm.GroupVersionKind().String()) -// } -// -// var oci Oci -// -// switch tm.Kind { -// case v1alpha1.FilesContentKind: -// var cfg v1alpha1.Files -// err := yaml.Unmarshal(data, &cfg) -// if err != nil { -// return nil, err -// } -// -// oci = file.New(cfg.Spec.Files[0].Name, cfg.Spec.Files[0].Ref) -// -// case v1alpha1.ImagesContentKind: -// var cfg v1alpha1.Images -// err := yaml.Unmarshal(data, &cfg) -// if err != nil { -// return nil, err -// } -// -// oci, err = image.New(cfg.Spec.Images[0].Ref) -// -// case v1alpha1.ChartsContentKind: -// var cfg v1alpha1.Charts -// err := yaml.Unmarshal(data, &cfg) -// if err != nil { -// return nil, err -// } -// -// oci = chart.New(cfg.Spec.Charts[0].Name, cfg.Spec.Charts[0].RepoURL, cfg.Spec.Charts[0].Version) -// -// case v1alpha1.DriverContentKind: -// var cfg v1alpha1.Driver -// err := yaml.Unmarshal(data, &cfg) -// if err != nil { -// return nil, err -// } -// -// return nil, fmt.Errorf("%s is still a wip", tm.GroupVersionKind().String()) -// -// default: -// return nil, fmt.Errorf("%s is not an understood content type", tm.GroupVersionKind().String()) -// } -// -// return oci, nil -// } -// -// // NewFromFile is a helper function around NewFromBytes to load new content given a filename -// func NewFromFile(filename string) ([]Oci, error) { -// fi, err := os.Open(filename) -// if err != nil { -// return nil, err -// } -// -// reader := yaml.NewYAMLReader(bufio.NewReader(fi)) -// -// var contents []Oci -// for { -// raw, err := reader.Read() -// if err == io.EOF { -// break -// } -// if err != nil { -// return nil, err -// } -// -// o, err := NewFromBytes(raw) -// if err != nil { -// return nil, err -// } -// -// contents = append(contents, o) -// } -// -// return contents, err -// } diff --git a/pkg/content/file/config.go b/pkg/content/file/config.go new file mode 100644 index 00000000..26863da2 --- /dev/null +++ b/pkg/content/file/config.go @@ -0,0 +1,82 @@ +package file + +import ( + "bytes" + "encoding/json" + + gv1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + gtypes "github.com/google/go-containerregistry/pkg/v1/types" + + "github.com/rancherfederal/hauler/pkg/artifact/types" +) + +var _ partial.Describable = (*config)(nil) + +type config struct { + Reference string `json:"ref"` // Reference is the reference from where the file was sourced + Name string `json:"name"` // Name is the files name on disk + Annotations map[string]string `json:"annotations,omitempty"` + URLs []string `json:"urls,omitempty"` + + computed bool + size int64 + hash gv1.Hash +} + +func (c config) Descriptor() (gv1.Descriptor, error) { + if err := c.compute(); err != nil { + return gv1.Descriptor{}, err + } + + return gv1.Descriptor{ + MediaType: types.FileMediaType, + Size: c.size, + Digest: c.hash, + URLs: c.URLs, + Annotations: c.Annotations, + // Platform: nil, + }, nil +} + +func (c config) Digest() (gv1.Hash, error) { + if err := c.compute(); err != nil { + return gv1.Hash{}, err + } + return c.hash, nil +} + +func (c config) MediaType() (gtypes.MediaType, error) { + return types.FileMediaType, nil +} + +func (c config) Size() (int64, error) { + if err := c.compute(); err != nil { + return 0, err + } + return c.size, nil +} + +func (c *config) Raw() ([]byte, error) { + return json.Marshal(c) +} + +func (c *config) compute() error { + if c.computed { + return nil + } + + data, err := c.Raw() + if err != nil { + return err + } + + h, size, err := gv1.SHA256(bytes.NewBuffer(data)) + if err != nil { + return err + } + + c.size = size + c.hash = h + return nil +} diff --git a/pkg/content/file/file.go b/pkg/content/file/file.go index 8c8003be..dc14f7bd 100644 --- a/pkg/content/file/file.go +++ b/pkg/content/file/file.go @@ -1,157 +1,107 @@ package file import ( - "context" "io" "net/http" - "net/url" "os" - "path" - "path/filepath" + "strings" - "github.com/containerd/containerd/remotes/docker" - "github.com/google/go-containerregistry/pkg/name" + gv1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + gtypes "github.com/google/go-containerregistry/pkg/v1/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" - orascontent "oras.land/oras-go/pkg/content" - "oras.land/oras-go/pkg/oras" - "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" - "github.com/rancherfederal/hauler/pkg/log" + "github.com/rancherfederal/hauler/pkg/artifact" + "github.com/rancherfederal/hauler/pkg/artifact/local" + "github.com/rancherfederal/hauler/pkg/artifact/types" ) -const ( - LayerMediaType = "application/vnd.hauler.cattle.io-artifact" -) - -type File struct { - cfg v1alpha1.File - - content getter -} - -func NewFile(cfg v1alpha1.File) File { - u, err := url.Parse(cfg.Ref) - if err != nil { - return File{content: local(cfg.Ref)} - } +var _ artifact.OCI = (*file)(nil) - var g getter - switch u.Scheme { - case "http", "https": - g = https{u} +type file struct { + blob gv1.Layer + config config + blobMap map[gv1.Hash]gv1.Layer - default: - g = local(cfg.Ref) - } - - return File{ - cfg: cfg, - content: g, - } + annotations map[string]string } -func (f File) Copy(ctx context.Context, registry string) error { - l := log.FromContext(ctx) - - resolver := docker.NewResolver(docker.ResolverOptions{}) - - // TODO: Should use a hybrid store that can mock out filenames - fs := orascontent.NewMemoryStore() - data, err := f.content.load() - if err != nil { - return err - } - - cname := f.content.name() - if f.cfg.Name != "" { - cname = f.cfg.Name - } - - desc := fs.Add(cname, f.content.mediaType(), data) - - ref, err := name.ParseReference(path.Join("hauler", cname), name.WithDefaultRegistry(registry)) - if err != nil { - return err - } - - l.Infof("Copying file to: %s", ref.Name()) - pushedDesc, err := oras.Push(ctx, resolver, ref.Name(), fs, []ocispec.Descriptor{desc}) - if err != nil { - return err +func NewFile(ref string, filename string) (artifact.OCI, error) { + var getter local.Opener + if strings.HasPrefix(ref, "http") || strings.HasPrefix(ref, "https") { + getter = remoteOpener(ref) + } else { + getter = localOpener(ref) } - l.Debugf("Copied with descriptor: %s", pushedDesc.Digest.String()) - return nil -} - -type getter interface { - load() ([]byte, error) - name() string - mediaType() string -} - -type local string + annotations := make(map[string]string) + annotations[ocispec.AnnotationTitle] = filename // For oras FileStore to recognize + annotations[ocispec.AnnotationSource] = ref -func (f local) load() ([]byte, error) { - fi, err := os.Stat(string(f)) + blob, err := local.LayerFromOpener(getter, + local.WithMediaType(types.FileLayerMediaType), + local.WithAnnotations(annotations)) if err != nil { return nil, err } - var data []byte - if fi.IsDir() { - data = []byte("") - } else { - data, err = os.ReadFile(string(f)) - if err != nil { - return nil, err - } + f := &file{ + blob: blob, + config: config{ + Reference: ref, + Name: filename, + }, } - - return data, nil + return f, nil } -func (f local) name() string { - return filepath.Base(string(f)) +func (f *file) MediaType() string { + return types.OCIManifestSchema1 } -func (f local) mediaType() string { - return "some-media-type" +func (f *file) RawConfig() ([]byte, error) { + return f.config.Raw() } -type https struct { - url *url.URL +func (f *file) Layers() ([]gv1.Layer, error) { + var layers []gv1.Layer + layers = append(layers, f.blob) + return layers, nil } -// TODO: Support auth -func (f https) load() ([]byte, error) { - resp, err := http.Get(f.url.String()) +func (f *file) Manifest() (*gv1.Manifest, error) { + desc, err := partial.Descriptor(f.blob) if err != nil { return nil, err } - defer resp.Body.Close() + layerDescs := []gv1.Descriptor{*desc} - data, err := io.ReadAll(resp.Body) + cfgDesc, err := f.config.Descriptor() if err != nil { return nil, err } - return data, nil + return &gv1.Manifest{ + SchemaVersion: 2, + MediaType: gtypes.MediaType(f.MediaType()), + Config: cfgDesc, + Layers: layerDescs, + Annotations: f.annotations, + }, nil } -func (f https) name() string { - resp, err := http.Get(f.url.String()) - if err != nil { - return "" - } - - switch resp.Header { - - default: - return path.Base(f.url.String()) +func localOpener(path string) local.Opener { + return func() (io.ReadCloser, error) { + return os.Open(path) } } -func (f https) mediaType() string { - return "some-remote-media-type" +func remoteOpener(url string) local.Opener { + return func() (io.ReadCloser, error) { + resp, err := http.Get(url) + if err != nil { + return nil, err + } + return resp.Body, nil + } } diff --git a/pkg/content/file/file_test.go b/pkg/content/file/file_test.go index 5de389a6..bc0fc49f 100644 --- a/pkg/content/file/file_test.go +++ b/pkg/content/file/file_test.go @@ -1,4 +1,4 @@ -package file +package file_test import ( "context" @@ -7,9 +7,17 @@ import ( "net/http/httptest" "os" "path/filepath" + "reflect" "testing" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" + "github.com/rancherfederal/hauler/pkg/artifact/types" + "github.com/rancherfederal/hauler/pkg/content/file" "github.com/rancherfederal/hauler/pkg/log" "github.com/rancherfederal/hauler/pkg/store" ) @@ -52,11 +60,11 @@ func TestFile_Copy(t *testing.T) { { name: "should copy a local file successfully without an explicit name", cfg: v1alpha1.File{ - Ref: f.Name(), + Ref: f.Name(), + Name: filepath.Base(f.Name()), }, args: args{ - ctx: ctx, - registry: s.RegistryURL(), + ctx: ctx, }, }, { @@ -66,8 +74,7 @@ func TestFile_Copy(t *testing.T) { Name: "my-other-file", }, args: args{ - ctx: ctx, - registry: s.RegistryURL(), + ctx: ctx, }, }, { @@ -77,8 +84,7 @@ func TestFile_Copy(t *testing.T) { Name: "my!invalid~@file", }, args: args{ - ctx: ctx, - registry: s.RegistryURL(), + ctx: ctx, }, wantErr: true, }, @@ -88,8 +94,7 @@ func TestFile_Copy(t *testing.T) { Ref: fmt.Sprintf("%s/%s", fs.server.URL, filepath.Base(f.Name())), }, args: args{ - ctx: ctx, - registry: s.RegistryURL(), + ctx: ctx, }, }, { @@ -99,18 +104,30 @@ func TestFile_Copy(t *testing.T) { Name: "my-other-file", }, args: args{ - ctx: ctx, - registry: s.RegistryURL(), + ctx: ctx, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - l.Debugf("doing: %s", tt.cfg.Ref) - f := NewFile(tt.cfg) - if err := f.Copy(tt.args.ctx, tt.args.registry); (err != nil) != tt.wantErr { - t.Errorf("Copy() error = %v, wantErr %v", err, tt.wantErr) + f, err := file.NewFile(tt.cfg.Ref, tt.cfg.Name) + if err != nil { + t.Fatal(err) + } + + ref, err := name.ParseReference(filepath.Base(tt.cfg.Ref)) + if err != nil { + t.Fatal(err) + } + + _, err = s.Add(ctx, f, ref) + if (err != nil) != tt.wantErr { + t.Error(err) } + + // if err := validate(tt.cfg.Ref, tt.cfg.Name, m); err != nil { + // t.Error(err) + // } }) } } @@ -132,3 +149,40 @@ func (s *testFileServer) Start() *httptest.Server { func (s *testFileServer) Stop() { s.server.Close() } + +// validate ensure +func validate(ref string, name string, got *v1.Manifest) error { + data, err := os.ReadFile(ref) + if err != nil { + return err + } + + d := digest.FromBytes(data) + + annotations := make(map[string]string) + annotations[ocispec.AnnotationTitle] = name + annotations[ocispec.AnnotationSource] = ref + + want := &v1.Manifest{ + SchemaVersion: 2, + MediaType: types.OCIManifestSchema1, + Config: v1.Descriptor{}, + Layers: []v1.Descriptor{ + { + MediaType: types.FileLayerMediaType, + Size: int64(len(data)), + Digest: v1.Hash{ + Algorithm: d.Algorithm().String(), + Hex: d.Hex(), + }, + Annotations: annotations, + }, + }, + Annotations: nil, + } + + if !reflect.DeepEqual(want.Layers, got.Layers) { + return fmt.Errorf("want = (%v) | got = (%v)", want, got) + } + return nil +} diff --git a/pkg/content/image/image.go b/pkg/content/image/image.go index 69709090..2d8de1cb 100644 --- a/pkg/content/image/image.go +++ b/pkg/content/image/image.go @@ -1,48 +1,43 @@ package image import ( - "context" - "github.com/google/go-containerregistry/pkg/name" + gv1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" - "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" - "github.com/rancherfederal/hauler/pkg/log" - "github.com/rancherfederal/hauler/pkg/store" + "github.com/rancherfederal/hauler/pkg/artifact" ) -type Image struct { - cfg v1alpha1.Image -} +var _ artifact.OCI = (*image)(nil) -func NewImage(cfg v1alpha1.Image) Image { - return Image{ - cfg: cfg, +func (i *image) MediaType() string { + mt, err := i.Image.MediaType() + if err != nil { + return "" } + return string(mt) } -func (i Image) Copy(ctx context.Context, registry string) error { - l := log.FromContext(ctx) +func (i *image) RawConfig() ([]byte, error) { + return i.RawConfigFile() +} - srcRef, err := name.ParseReference(i.cfg.Ref) - if err != nil { - return err - } +type image struct { + gv1.Image +} - img, err := remote.Image(srcRef) +func NewImage(ref string) (artifact.OCI, error) { + r, err := name.ParseReference(ref) if err != nil { - return err + return nil, err } - dstRef, err := store.RelocateReference(srcRef, registry) + img, err := remote.Image(r) if err != nil { - return err - } - - l.Infof("Copying image to: '%s'", dstRef.Name()) - if err := remote.Write(dstRef, img, remote.WithContext(ctx)); err != nil { - return err + return nil, err } - return nil + return &image{ + Image: img, + }, nil } diff --git a/pkg/content/image/image_test.go b/pkg/content/image/image_test.go index f281870a..2b9ff662 100644 --- a/pkg/content/image/image_test.go +++ b/pkg/content/image/image_test.go @@ -1,11 +1,16 @@ -package image +package image_test import ( "context" "os" + "path" + "path/filepath" "testing" + "github.com/google/go-containerregistry/pkg/name" + "github.com/rancherfederal/hauler/pkg/apis/hauler.cattle.io/v1alpha1" + "github.com/rancherfederal/hauler/pkg/content/image" "github.com/rancherfederal/hauler/pkg/log" "github.com/rancherfederal/hauler/pkg/store" ) @@ -42,8 +47,8 @@ func TestImage_Copy(t *testing.T) { Ref: "busybox:1.34.1", }, args: args{ - ctx: ctx, - registry: s.RegistryURL(), + ctx: ctx, + // registry: s.Registry(), }, wantErr: false, }, @@ -53,19 +58,42 @@ func TestImage_Copy(t *testing.T) { Ref: "busybox@sha256:6066ca124f8c2686b7ae71aa1d6583b28c6dc3df3bdc386f2c89b92162c597d9", }, args: args{ - ctx: ctx, - registry: s.RegistryURL(), + ctx: ctx, + // registry: s.Registry(), + }, + wantErr: false, + }, + { + name: "should work with tagged image", + cfg: v1alpha1.Image{ + Ref: "registry:2", + }, + args: args{ + ctx: ctx, + // registry: s.Registry(), }, wantErr: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - i := NewImage(tt.cfg) + i, err := image.NewImage(tt.cfg.Ref) + if err != nil { + t.Error(err) + } - if err := i.Copy(tt.args.ctx, tt.args.registry); (err != nil) != tt.wantErr { - t.Errorf("Copy() error = %v, wantErr %v", err, tt.wantErr) + ref, err := name.ParseReference(path.Join("hauler", filepath.Base(tt.cfg.Ref))) + if err != nil { + t.Fatal(err) } + + if _, err := s.Add(ctx, i, ref); (err != nil) != tt.wantErr { + t.Error(err) + } + + // if err := s.Add(tt.args.ctx, i, ref); (err != nil) != tt.wantErr { + // t.Errorf("Copy() error = %v, wantErr %v", err, tt.wantErr) + // } }) } } diff --git a/pkg/content/k3s/k3s.go b/pkg/content/k3s/k3s.go deleted file mode 100644 index a3acc0ff..00000000 --- a/pkg/content/k3s/k3s.go +++ /dev/null @@ -1,53 +0,0 @@ -package k3s - -import ( - "context" - - "github.com/rancherfederal/hauler/pkg/content/file" - "github.com/rancherfederal/hauler/pkg/content/image" -) - -type K3s struct { - Files []file.File - Images []image.Image -} - -func NewK3s(version string) (*K3s, error) { - bom, err := newDependencies("k3s", version) - if err != nil { - return nil, err - } - - var files []file.File - for _, f := range bom.files.Spec.Files { - fi := file.NewFile(f) - files = append(files, fi) - } - - var images []image.Image - for _, i := range bom.images.Spec.Images { - img := image.NewImage(i) - images = append(images, img) - } - - return &K3s{ - Files: files, - Images: images, - }, nil -} - -func (k *K3s) Copy(ctx context.Context, registry string) error { - for _, f := range k.Files { - if err := f.Copy(ctx, registry); err != nil { - return err - } - } - - for _, i := range k.Images { - if err := i.Copy(ctx, registry); err != nil { - return err - } - } - - return nil -} diff --git a/pkg/layout/artifact.go b/pkg/layout/artifact.go new file mode 100644 index 00000000..5e17ceee --- /dev/null +++ b/pkg/layout/artifact.go @@ -0,0 +1,132 @@ +package layout + +import ( + "bytes" + "encoding/json" + "io" + "os" + + gv1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/layout" + gtypes "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "golang.org/x/sync/errgroup" + + "github.com/rancherfederal/hauler/pkg/artifact" +) + +type Path struct { + layout.Path +} + +func FromPath(path string) (Path, error) { + p, err := layout.FromPath(path) + if os.IsNotExist(err) { + p, err = layout.Write(path, empty.Index) + if err != nil { + return Path{}, err + } + } + return Path{Path: p}, err +} + +func (l Path) WriteOci(o artifact.OCI, name string) (ocispec.Descriptor, error) { + layers, err := o.Layers() + if err != nil { + return ocispec.Descriptor{}, err + } + + // Write layers concurrently + var g errgroup.Group + for _, layer := range layers { + layer := layer + g.Go(func() error { + return l.writeLayer(layer) + }) + } + if err := g.Wait(); err != nil { + return ocispec.Descriptor{}, err + } + + // Write the config + cfgBlob, err := o.RawConfig() + if err != nil { + return ocispec.Descriptor{}, err + } + + if err = l.writeBlob(cfgBlob); err != nil { + return ocispec.Descriptor{}, err + } + + m, err := o.Manifest() + if err != nil { + return ocispec.Descriptor{}, err + } + + manifest, err := json.Marshal(m) + if err != nil { + return ocispec.Descriptor{}, err + } + + if err := l.writeBlob(manifest); err != nil { + return ocispec.Descriptor{}, err + } + + desc := ocispec.Descriptor{ + MediaType: o.MediaType(), + Size: int64(len(manifest)), + Digest: digest.FromBytes(manifest), + Annotations: map[string]string{ + ocispec.AnnotationRefName: name, + }, + } + + if err := l.appendDescriptor(desc); err != nil { + return ocispec.Descriptor{}, err + } + + return desc, nil +} + +// writeBlob differs from layer.WriteBlob in that it requires data instead +func (l Path) writeBlob(data []byte) error { + h, _, err := gv1.SHA256(bytes.NewReader(data)) + if err != nil { + return err + } + + return l.WriteBlob(h, io.NopCloser(bytes.NewReader(data))) +} + +// writeLayer is a verbatim reimplementation of layout.writeLayer +func (l Path) writeLayer(layer gv1.Layer) error { + d, err := layer.Digest() + if err != nil { + return err + } + + r, err := layer.Compressed() + if err != nil { + return err + } + + return l.WriteBlob(d, r) +} + +// appendDescriptor is a helper that translates a ocispec.Descriptor into a gv1.Descriptor +func (l Path) appendDescriptor(desc ocispec.Descriptor) error { + gdesc := gv1.Descriptor{ + MediaType: gtypes.MediaType(desc.MediaType), + Size: desc.Size, + Digest: gv1.Hash{ + Algorithm: desc.Digest.Algorithm().String(), + Hex: desc.Digest.Hex(), + }, + URLs: desc.URLs, + Annotations: desc.Annotations, + } + + return l.AppendDescriptor(gdesc) +} diff --git a/pkg/layout/store.go b/pkg/layout/store.go new file mode 100644 index 00000000..15c851e8 --- /dev/null +++ b/pkg/layout/store.go @@ -0,0 +1,165 @@ +package layout + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/content/local" + "github.com/containerd/containerd/remotes/docker" + "github.com/google/go-containerregistry/pkg/name" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + orascontent "oras.land/oras-go/pkg/content" + "oras.land/oras-go/pkg/oras" + + "github.com/rancherfederal/hauler/pkg/artifact/types" +) + +type OCIStore struct { + content.Store + + root string + index *ocispec.Index + digestMap map[string]ocispec.Descriptor +} + +// Copy placeholder until we migrate to oras 0.5 +// Will loop through each appropriately named index and copy the contents to the desired registry +func Copy(ctx context.Context, s *OCIStore, registry string) error { + for _, desc := range s.index.Manifests { + manifestBlobPath, err := s.blobPath(desc.Digest) + if err != nil { + return err + } + + manifestData, err := os.ReadFile(manifestBlobPath) + if err != nil { + return err + } + + m, mdesc, err := loadManifest(manifestData) + if err != nil { + return err + } + + refName, ok := desc.Annotations[ocispec.AnnotationRefName] + if !ok { + return fmt.Errorf("no name found to push image") + } + + ref, err := name.ParseReference(refName, name.WithDefaultRegistry(registry)) + if err != nil { + return err + } + + resolver := docker.NewResolver(docker.ResolverOptions{}) + _, err = oras.Push(ctx, resolver, ref.Name(), s, m.Layers, + oras.WithConfig(m.Config), oras.WithNameValidation(nil), oras.WithManifest(mdesc)) + + if err != nil { + return err + } + } + + return nil +} + +func NewOCIStore(rootPath string) (*OCIStore, error) { + fs, err := local.NewStore(rootPath) + if err != nil { + return nil, err + } + + store := &OCIStore{ + Store: fs, + + root: rootPath, + } + + if err := store.validateOCILayout(); err != nil { + return nil, err + } + if err := store.LoadIndex(); err != nil { + return nil, nil + } + + return store, nil +} + +func (s *OCIStore) LoadIndex() error { + path := filepath.Join(s.root, types.OCIImageIndexFile) + indexFile, err := os.Open(path) + if err != nil { + // TODO: Don't just bomb out? + return err + } + defer indexFile.Close() + + if err := json.NewDecoder(indexFile).Decode(&s.index); err != nil { + return err + } + + s.digestMap = make(map[string]ocispec.Descriptor) + for _, desc := range s.index.Manifests { + if name := desc.Annotations[ocispec.AnnotationRefName]; name != "" { + s.digestMap[name] = desc + } + } + + return nil +} + +func (s *OCIStore) validateOCILayout() error { + layoutFilePath := filepath.Join(s.root, ocispec.ImageLayoutFile) + layoutFile, err := os.Open(layoutFilePath) + if err != nil { + return err + } + defer layoutFile.Close() + + var layout *ocispec.ImageLayout + if err := json.NewDecoder(layoutFile).Decode(&layout); err != nil { + return err + } + + if layout.Version != ocispec.ImageLayoutVersion { + return orascontent.ErrUnsupportedVersion + } + + return nil +} + +func (s *OCIStore) blobPath(d digest.Digest) (string, error) { + if err := d.Validate(); err != nil { + return "", err + } + + return filepath.Join(s.root, "blobs", d.Algorithm().String(), d.Hex()), nil +} + +// manifest is a field wrapper around ocispec.Manifest that contains the mediaType field +type manifest struct { + ocispec.Manifest `json:",inline"` + + MediaType string `json:"mediaType"` +} + +// loadManifest +func loadManifest(data []byte) (ocispec.Manifest, ocispec.Descriptor, error) { + var m manifest + if err := json.Unmarshal(data, &m); err != nil { + return ocispec.Manifest{}, ocispec.Descriptor{}, err + } + + desc := ocispec.Descriptor{ + MediaType: m.MediaType, + Digest: digest.FromBytes(data), + Size: int64(len(data)), + } + + return m.Manifest, desc, nil +} diff --git a/pkg/store/add.go b/pkg/store/add.go index 5916c627..0ce78f89 100644 --- a/pkg/store/add.go +++ b/pkg/store/add.go @@ -2,46 +2,91 @@ package store import ( "context" + "io/ioutil" + "os" + "path/filepath" - "github.com/rancherfederal/hauler/pkg/content" + "github.com/google/go-containerregistry/pkg/name" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + "github.com/rancherfederal/hauler/pkg/artifact" + "github.com/rancherfederal/hauler/pkg/layout" "github.com/rancherfederal/hauler/pkg/log" ) -type addOptions struct { - repo string -} +// Add will add an artifact.OCI to the store +// The method to achieve this is to save artifact.OCI to a temporary directory in an OCI layout compatible form. Once +// saved, the entirety of the layout is copied to the store (which is just a registry). This allows us to not only use +// strict types to define generic content, but provides a processing pipeline suitable for extensibility. In the +// future we'll allow users to define their own content that must adhere either by artifact.OCI or simply an OCI layout. +func (s *Store) Add(ctx context.Context, oci artifact.OCI, locationRef name.Reference) (ocispec.Descriptor, error) { + lgr := log.FromContext(ctx) -type AddOption func(*addOptions) + if err := s.precheck(); err != nil { + return ocispec.Descriptor{}, err + } -func makeAddOptions(opts ...AddOption) addOptions { - opt := addOptions{} - for _, o := range opts { - o(&opt) + relocated, err := RelocateReference(locationRef, s.Registry()) + if err != nil { + return ocispec.Descriptor{}, err } - return opt -} -func (s *Store) Add(ctx context.Context, oci content.Oci, opts ...AddOption) error { - l := log.FromContext(ctx) - opt := makeAddOptions(opts...) + lgr.Debugf("adding %s to store", relocated.Name()) - if err := s.precheck(); err != nil { - return err + tmpdir, err := os.MkdirTemp("", "hauler") + if err != nil { + return ocispec.Descriptor{}, err + } + defer os.RemoveAll(tmpdir) + + l, err := layout.FromPath(tmpdir) + if err != nil { + return ocispec.Descriptor{}, err } - if opt.repo == "" { + mdesc, err := l.WriteOci(oci, relocated.Name()) + if err != nil { + return ocispec.Descriptor{}, err } - if err := oci.Copy(ctx, s.RegistryURL()); err != nil { + if err := s.AddFromLayout(ctx, tmpdir); err != nil { + return ocispec.Descriptor{}, err + } + + return mdesc, err +} + +// AddFromLayout will read an oci-layout and add all manifests referenced in index.json to the store +func (s *Store) AddFromLayout(ctx context.Context, layoutPath string) error { + if err := s.precheck(); err != nil { return err } - _ = l - return nil + ociStore, err := layout.NewOCIStore(layoutPath) + if err != nil { + return err + } + + return layout.Copy(ctx, ociStore, s.Registry()) } -func OverrideRepo(r string) AddOption { - return func(opts *addOptions) { - opts.repo = r +// Flush is a fancy name for delete-all-the-things, in this case it's as trivial as deleting everything in the underlying store directory +// This can be a highly destructive operation if the store's directory happens to be inline with other non-store contents +// To reduce the blast radius and likelihood of deleting things we don't own, Flush explicitly includes docker/registry/v2 +// in the search dir +func (s *Store) Flush(ctx context.Context) error { + contentDir := filepath.Join(s.DataDir, "docker", "registry", "v2") + fs, err := ioutil.ReadDir(contentDir) + if err != nil { + return err + } + + for _, f := range fs { + err := os.RemoveAll(filepath.Join(contentDir, f.Name())) + if err != nil { + return err + } } + + return nil } diff --git a/pkg/store/store.go b/pkg/store/store.go index 37f78654..b425379f 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -3,26 +3,25 @@ package store import ( "context" "fmt" + "io" "net/http" "net/http/httptest" "regexp" + "strconv" "time" "github.com/distribution/distribution/v3/configuration" dcontext "github.com/distribution/distribution/v3/context" + "github.com/distribution/distribution/v3/reference" + "github.com/distribution/distribution/v3/registry/client" "github.com/distribution/distribution/v3/registry/handlers" _ "github.com/distribution/distribution/v3/registry/storage/driver/filesystem" "github.com/google/go-containerregistry/pkg/name" "github.com/sirupsen/logrus" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/rancherfederal/hauler/pkg/content" ) var ( httpRegex = regexp.MustCompile("https?://") - - contents = make(map[metav1.TypeMeta]content.Oci) ) // Store is a simple wrapper around distribution/distribution to enable hauler's use case @@ -61,17 +60,6 @@ func NewStore(ctx context.Context, dataDir string) *Store { } } -// TODO: Refactor to a feature register model for content types -func Register(gvk metav1.TypeMeta, oci content.Oci) { - if oci == nil { - panic("store: Register content is nil") - } - if _, dup := contents[gvk]; dup { - panic("store: Register called twice for content " + gvk.String()) - } - contents[gvk] = oci -} - // Open will create a new server and start it, it's up to the consumer to close it func (s *Store) Open() *httptest.Server { server := httptest.NewServer(s.handler) @@ -93,16 +81,18 @@ func (s *Store) Remove() error { return nil } -func RelocateReference(ref name.Reference, registry string) (name.Reference, error) { +func RelocateReference(ref name.Reference, registry string, opts ...name.Option) (name.Reference, error) { var sep string - if _, err := name.NewDigest(ref.Name()); err == nil { + if _, err := name.NewDigest(ref.Name(), opts...); err == nil { sep = "@" } else { sep = ":" } + + opts = append(opts, name.WithDefaultRegistry(registry)) return name.ParseReference( fmt.Sprintf("%s%s%s", ref.Context().RepositoryStr(), sep, ref.Identifier()), - name.WithDefaultRegistry(registry), + opts..., ) } @@ -115,11 +105,68 @@ func (s *Store) RelocateReference(ref name.Reference) name.Reference { } relocatedRef, _ := name.ParseReference( fmt.Sprintf("%s%s%s", ref.Context().RepositoryStr(), sep, ref.Identifier()), - name.WithDefaultRegistry(s.RegistryURL()), + name.WithDefaultRegistry(s.Registry()), ) return relocatedRef } +// List will list all known content tags in the registry +// TODO: This fn is messy and needs cleanup, this is arguably easier with the catalog api as well +func (s *Store) List(ctx context.Context) ([]string, error) { + reg, err := client.NewRegistry(s.RegistryURL(), nil) + if err != nil { + return nil, err + } + + entries := make(map[string]reference.Named) + last := "" + for { + chunk := make([]string, 20) // randomly chosen number... + nf, err := reg.Repositories(ctx, chunk, last) + last = strconv.Itoa(nf) + + for _, e := range chunk { + if e == "" { + continue + } + + ref, err := reference.WithName(e) + if err != nil { + return nil, err + } + entries[e] = ref + } + if err == io.EOF { + break + } + } + + var refs []string + for ref, named := range entries { + repo, err := client.NewRepository(named, s.RegistryURL(), nil) + if err != nil { + return nil, err + } + + tsvc := repo.Tags(ctx) + + ts, err := tsvc.All(ctx) + if err != nil { + return nil, err + } + + for _, t := range ts { + ref, err := name.ParseReference(ref, name.WithDefaultRegistry(""), name.WithDefaultTag(t)) + if err != nil { + return nil, err + } + refs = append(refs, ref.Name()) + } + } + + return refs, nil +} + // precheck checks whether server is appropriately started and errors if it's not // used to safely run Store operations without fear of panics func (s *Store) precheck() error { @@ -129,11 +176,16 @@ func (s *Store) precheck() error { return nil } -// RegistryURL returns the registries URL without the protocol, suitable for image relocation operations -func (s *Store) RegistryURL() string { +// Registry returns the registries URL without the protocol, suitable for image relocation operations +func (s *Store) Registry() string { return httpRegex.ReplaceAllString(s.server.URL, "") } +// RegistryURL returns the registries URL +func (s *Store) RegistryURL() string { + return s.server.URL +} + func alive(path string, handler http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == path { diff --git a/testdata/contents.yaml b/testdata/contents.yaml index 57165f48..c299055a 100644 --- a/testdata/contents.yaml +++ b/testdata/contents.yaml @@ -8,7 +8,7 @@ spec: - ref: testdata/contents.yaml # when directories are specified, they will be archived and stored as a file - - ref: testdata/ +# - ref: testdata/ # hauler can also fetch remote content, and will "smartly" identify filenames _when possible_ # filename below = "k3s-airgap-images-arm64.tar.zst" @@ -39,26 +39,6 @@ metadata: spec: charts: # charts are also fetched and served as OCI content (currently experimental in helm) - # HELM_EXPERIMENTAL_OCI=1 helm chart pull /hauler/rancher:2.6.2 - - name: rancher - repoURL: https://releases.rancher.com/server-charts/latest - version: 2.6.2 - ---- -apiVersion: content.hauler.cattle.io/v1alpha1 -kind: Driver -metadata: - name: mydriver -spec: - type: k3s - version: v1.22.2+k3s2 - - ---- -apiVersion: collection.hauler.cattle.io/v1alpha1 -kind: K3s -metadata: - name: mycollection -spec: - version: 0.1.0 - + # HELM_EXPERIMENTAL_OCI=1 helm chart pull /loki:2.6.2 + - name: loki + repoURL: https://grafana.github.io/helm-charts