From 7673fcecbdebf5ec2930fbbb5472251fc261e20d Mon Sep 17 00:00:00 2001 From: Dayuan Date: Thu, 18 Apr 2024 21:16:12 +0800 Subject: [PATCH] feat: group multiple os arch modules by image index https://github.com/opencontainers/image-spec/blob/main/image-index.md --- pkg/cmd/mod/mod_push.go | 64 ++++++++++++++----------- pkg/oci/client/push.go | 102 +++++++++++++++++++++++++++++++++------- 2 files changed, 122 insertions(+), 44 deletions(-) diff --git a/pkg/cmd/mod/mod_push.go b/pkg/cmd/mod/mod_push.go index 3c77e5f5..ea26cf6d 100644 --- a/pkg/cmd/mod/mod_push.go +++ b/pkg/cmd/mod/mod_push.go @@ -131,7 +131,7 @@ func NewCmdPush(ioStreams genericiooptions.IOStreams) *cobra.Command { func (flags *PushModFlags) AddFlags(cmd *cobra.Command) { cmd.Flags().StringVarP(&flags.Version, "version", "v", "", "The version of the module e.g. '1.0.0' or '1.0.0-rc.1'.") cmd.Flags().StringVar(&flags.OSArch, "os-arch", "", "The os arch of the module e.g. 'darwin/arm64', 'linux/amd64'.") - cmd.Flags().BoolVar(&flags.Latest, "latest", flags.Latest, "Tags the current version as the latest stable module version.") + cmd.Flags().BoolVar(&flags.Latest, "latest", true, "Tags the current version as the latest stable module version.") cmd.Flags().StringVar(&flags.Credentials, "creds", flags.Credentials, "The credentials token for the OCI registry in or : format.") cmd.Flags().StringVar(&flags.Sign, "sign", flags.Sign, "Signs the module with the specified provider.") @@ -147,18 +147,11 @@ func (flags *PushModFlags) ToOptions(args []string, ioStreams genericiooptions.I return nil, fmt.Errorf("path to module and OCI registry url are required") } - // Prepare metadata - if flags.OSArch == "" { - // set as the current OS arch - flags.OSArch = runtime.GOOS + "/" + runtime.GOARCH - } - osArch := strings.Split(flags.OSArch, "/") - + // version version := flags.Version if _, err := semver.StrictNewVersion(version); err != nil { return nil, fmt.Errorf("version is not in semver format: %w", err) } - fullURL := fmt.Sprintf("%s-%s_%s:%s", args[1], osArch[0], osArch[1], version) // If creds in format, creds must be base64 encoded if len(flags.Credentials) != 0 && !strings.Contains(flags.Credentials, ":") { @@ -179,6 +172,13 @@ func (flags *PushModFlags) ToOptions(args []string, ioStreams genericiooptions.I info = gitutil.Get(repoRoot) } + // os arch + if flags.OSArch == "" { + // set as the current OS arch + flags.OSArch = runtime.GOOS + "/" + runtime.GOARCH + } + osArch := strings.Split(flags.OSArch, "/") + meta := metadata.Metadata{ Created: info.CommitDate, Source: info.RemoteURL, @@ -205,12 +205,13 @@ func (flags *PushModFlags) ToOptions(args []string, ioStreams genericiooptions.I opt := &PushModOptions{ ModulePath: args[0], - OCIUrl: fullURL, + OCIUrl: args[1], Latest: flags.Latest, Sign: flags.Sign, CosignKey: flags.CosignKey, Client: client, Metadata: meta, + Version: version, IOStreams: ioStreams, } @@ -261,28 +262,36 @@ func (o *PushModOptions) Run() error { } sp.Info("pushing the module...") - digest, err := o.Client.Push(ctx, o.OCIUrl, targetDir, o.Metadata, nil) + idxDigestURL, imgDigestURL, err := o.Client.Push(ctx, o.OCIUrl, o.Version, targetDir, o.Metadata, nil) if err != nil { return err } // Tag latest version if required if o.Latest { - if err = o.Client.Tag(ctx, digest, LatestVersion); err != nil { - return fmt.Errorf("tagging module version as latest failed: %w", err) + if err = o.Client.Tag(ctx, imgDigestURL, LatestVersion); err != nil { + return fmt.Errorf("tagging module image version as latest failed: %w", err) + } + if err = o.Client.Tag(ctx, idxDigestURL, LatestVersion); err != nil { + return fmt.Errorf("tagging module index version as latest failed: %w", err) } } - sp.Info("pushed successfully\n") - _ = sp.Stop() // Signs the module with specific provider if len(o.Sign) != 0 { - err = oci.SignArtifact(o.Sign, digest, o.CosignKey) + err = oci.SignArtifact(o.Sign, imgDigestURL, o.CosignKey) + if err != nil { + return err + } + err = oci.SignArtifact(o.Sign, idxDigestURL, o.CosignKey) if err != nil { return err } } + sp.Info("pushed successfully\n") + _ = sp.Stop() + return nil } @@ -298,11 +307,16 @@ func (o *PushModOptions) buildModule() (string, error) { moduleSrc := filepath.Join(o.ModulePath, "src") goFileSearchPattern := filepath.Join(moduleSrc, "*.go") - // OCIUrl example: oci://ghcr.io/org/my-module-linux_amd64:0.1.0 + // prepare platform + if o.Metadata.Platform == nil { + return "", fmt.Errorf("platform is not set in metadata") + } + pOS := o.Metadata.Platform.OS + pArch := o.Metadata.Platform.Architecture + + // OCIUrl example: oci://ghcr.io/org/my-module split := strings.Split(o.OCIUrl, "/") - nameVersion := strings.Split(split[len(split)-1], ":") - name := nameVersion[0] - version := nameVersion[1] + name := split[len(split)-1] if matches, err := filepath.Glob(goFileSearchPattern); err != nil || len(matches) == 0 { return "", fmt.Errorf("no go source code files found for 'go build' matching %s", goFileSearchPattern) @@ -313,15 +327,9 @@ func (o *PushModOptions) buildModule() (string, error) { return "", fmt.Errorf("unable to find executable 'go' binary: %w", err) } - // prepare platform - if o.Metadata.Platform == nil { - return "", fmt.Errorf("platform is not set in metadata") - } - pOS := o.Metadata.Platform.OS - pArch := o.Metadata.Platform.Architecture - output := filepath.Join(targetDir, "_dist", pOS, pArch, "kusion-module-"+name+"_"+version) + output := filepath.Join(targetDir, "_dist", pOS, pArch, "kusion-module-"+name+"_"+o.Version) if strings.Contains(o.OSArch, "windows") { - output = filepath.Join(targetDir, "_dist", pOS, pArch, "kusion-module-"+name+"_"+version+".exe") + output = filepath.Join(targetDir, "_dist", pOS, pArch, "kusion-module-"+name+"_"+o.Version+".exe") } path, err := buildBinary(goBin, pOS, pArch, moduleSrc, output, o.IOStreams) diff --git a/pkg/oci/client/push.go b/pkg/oci/client/push.go index 858ec1f4..28ff210c 100644 --- a/pkg/oci/client/push.go +++ b/pkg/oci/client/push.go @@ -2,6 +2,7 @@ package client import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -11,6 +12,9 @@ import ( "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/google/go-containerregistry/pkg/v1/types" @@ -25,23 +29,57 @@ const ArtifactTarballFileName = "artifact.tgz" // - builds tarball from given directory also corresponding layer // - adds this layer to an empty OpenContainers artifact // - annotates the artifact with the given annotations -// - uploads the final artifact to the OCI registry +// - check if the target reference exists and if it is an image index. Creates a new image index if it does not exist +// - uploads the final artifact to the OCI registry and appends the artifact to the index, // - returns the digest URL of the upstream artifact -func (c *Client) Push(ctx context.Context, ociURL, sourceDir string, metadata meta.Metadata, ignorePaths []string) (string, error) { +func (c *Client) Push( + ctx context.Context, + ociURL, version, sourceDir string, + metadata meta.Metadata, + ignorePaths []string, +) (string, string, error) { ref, err := oci.ParseArtifactRef(ociURL) if err != nil { - return "", fmt.Errorf("invalid OCI repository url: %w", err) + return "", "", fmt.Errorf("invalid OCI repository url: %w", err) } + // Check if the target reference exists and if it is an image index + refStr := ref.String() + exists := true + var base v1.ImageIndex + + manifest, err := crane.Get(refStr, c.opts.craneOptions...) + if err != nil { + var t *transport.Error + ok := errors.As(err, &t) + if ok && t.StatusCode == 404 { + exists = false + base = empty.Index + } else { + return "", "", fmt.Errorf("get manifest failed: %s, %w", refStr, err) + } + } + + if exists { + if !manifest.MediaType.IsIndex() { + return "", "", fmt.Errorf("expected %s to be an index, got %q", refStr, manifest.MediaType) + } + base, err = manifest.ImageIndex() + if err != nil { + return "", "", fmt.Errorf("get manifest image index failed: %s, %w", refStr, err) + } + } + + // build image tmpDir, err := os.MkdirTemp("", "oci") if err != nil { - return "", err + return "", "", err } defer os.RemoveAll(tmpDir) tmpFile := filepath.Join(tmpDir, ArtifactTarballFileName) - if err := c.Build(tmpFile, sourceDir, ignorePaths); err != nil { - return "", err + if err = c.Build(tmpFile, sourceDir, ignorePaths); err != nil { + return "", "", err } // Add missing metadata @@ -56,19 +94,19 @@ func (c *Client) Push(ctx context.Context, ociURL, sourceDir string, metadata me platform := metadata.Platform if platform == nil { - return "", fmt.Errorf("platform is not set") + return "", "", fmt.Errorf("platform is not set") } image, err = mutate.ConfigFile(image, &v1.ConfigFile{ Architecture: platform.Architecture, OS: platform.OS, }) if err != nil { - return "", fmt.Errorf("setting image config file failed: %w", err) + return "", "", fmt.Errorf("setting image config file failed: %w", err) } layer, err := tarball.LayerFromFile(tmpFile, tarball.WithMediaType(CanonicalContentMediaType)) if err != nil { - return "", fmt.Errorf("creating content layer failed: %w", err) + return "", "", fmt.Errorf("creating content layer failed: %w", err) } image, err = mutate.Append(image, mutate.Addendum{ @@ -78,18 +116,50 @@ func (c *Client) Push(ctx context.Context, ociURL, sourceDir string, metadata me }, }) if err != nil { - return "", fmt.Errorf("appeding content to artifact failed: %w", err) + return "", "", fmt.Errorf("appeding content to artifact failed: %w", err) + } + + imgURL := fmt.Sprintf("%s-%s_%s:%s", ociURL, platform.OS, platform.Architecture, version) + imgRef, err := oci.ParseArtifactRef(imgURL) + if err != nil { + return "", "", fmt.Errorf("invalid image repository url: %w", err) + } + if err = crane.Push(image, imgRef.String(), c.optionsWithContext(ctx)...); err != nil { + return "", "", fmt.Errorf("pushing artifact failed: %w", err) + } + imgDigest, err := image.Digest() + if err != nil { + return "", "", fmt.Errorf("parsing image digest failed: %w", err) } - if err := crane.Push(image, ref.String(), c.optionsWithContext(ctx)...); err != nil { - return "", fmt.Errorf("pushing artifact failed: %w", err) + cf, err := image.ConfigFile() + if err != nil { + return "", "", fmt.Errorf("parsing image config file failed: %w", err) } - digest, err := image.Digest() + newDesc, err := partial.Descriptor(image) if err != nil { - return "", fmt.Errorf("parsing artifact digest failed: %w", err) + return "", "", fmt.Errorf("parsing image descriptor file failed: %w", err) + } + newDesc.Platform = cf.Platform() + addendum := mutate.IndexAddendum{ + Add: image, + Descriptor: *newDesc, + } + idx := mutate.AppendManifests(base, addendum) + idxDigest, err := idx.Digest() + if err != nil { + return "", "", fmt.Errorf("parsing index digest failed: %w", err) + } + + o := crane.GetOptions(c.opts.craneOptions...) + if err = remote.WriteIndex(ref, idx, o.Remote...); err != nil { + return "", "", fmt.Errorf("pushing image index %s: %w", refStr, err) } - digestURL := ref.Context().Digest(digest.String()).String() - return fmt.Sprintf("%s%s", oci.OCIRepositoryPrefix, digestURL), nil + idxDigestURL := ref.Context().Digest(idxDigest.String()).String() + imgDigestURL := ref.Context().Digest(imgDigest.String()).String() + idxURL := fmt.Sprintf("%s%s", oci.OCIRepositoryPrefix, idxDigestURL) + imgURL = fmt.Sprintf("%s%s", oci.OCIRepositoryPrefix, imgDigestURL) + return idxURL, imgURL, nil }