Skip to content

Commit

Permalink
feat: group multiple os arch modules by image index (#1054)
Browse files Browse the repository at this point in the history 
https://github.com/opencontainers/image-spec/blob/main/image-index.md
  • Loading branch information
@SparkYuan
SparkYuan committed Apr 19, 2024
1 parent 691640b commit 41a90a4
Show file tree
Hide file tree
Showing 2 changed files with 122 additions and 44 deletions.
64 changes: 36 additions & 28 deletions pkg/cmd/mod/mod_push.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ func NewCmdPush(ioStreams genericiooptions.IOStreams) *cobra.Command {
func (flags *PushModFlags) AddFlags(cmd *cobra.Command) {
cmd.Flags().StringVarP(&flags.Version, "version", "v", "", "The version of the module e.g. '1.0.0' or '1.0.0-rc.1'.")
cmd.Flags().StringVar(&flags.OSArch, "os-arch", "", "The os arch of the module e.g. 'darwin/arm64', 'linux/amd64'.")
cmd.Flags().BoolVar(&flags.Latest, "latest", flags.Latest, "Tags the current version as the latest stable module version.")
cmd.Flags().BoolVar(&flags.Latest, "latest", true, "Tags the current version as the latest stable module version.")
cmd.Flags().StringVar(&flags.Credentials, "creds", flags.Credentials,
"The credentials token for the OCI registry in <YOUR_TOKEN> or <YOUR_USERNAME>:<YOUR_TOKEN> format.")
cmd.Flags().StringVar(&flags.Sign, "sign", flags.Sign, "Signs the module with the specified provider.")
Expand All @@ -147,18 +147,11 @@ func (flags *PushModFlags) ToOptions(args []string, ioStreams genericiooptions.I
return nil, fmt.Errorf("path to module and OCI registry url are required")
}

// Prepare metadata
if flags.OSArch == "" {
// set as the current OS arch
flags.OSArch = runtime.GOOS + "/" + runtime.GOARCH
}
osArch := strings.Split(flags.OSArch, "/")

// version
version := flags.Version
if _, err := semver.StrictNewVersion(version); err != nil {
return nil, fmt.Errorf("version is not in semver format: %w", err)
}
fullURL := fmt.Sprintf("%s-%s_%s:%s", args[1], osArch[0], osArch[1], version)

// If creds in <token> format, creds must be base64 encoded
if len(flags.Credentials) != 0 && !strings.Contains(flags.Credentials, ":") {
Expand All @@ -179,6 +172,13 @@ func (flags *PushModFlags) ToOptions(args []string, ioStreams genericiooptions.I
info = gitutil.Get(repoRoot)
}

// os arch
if flags.OSArch == "" {
// set as the current OS arch
flags.OSArch = runtime.GOOS + "/" + runtime.GOARCH
}
osArch := strings.Split(flags.OSArch, "/")

meta := metadata.Metadata{
Created: info.CommitDate,
Source: info.RemoteURL,
Expand All @@ -205,12 +205,13 @@ func (flags *PushModFlags) ToOptions(args []string, ioStreams genericiooptions.I

opt := &PushModOptions{
ModulePath: args[0],
OCIUrl: fullURL,
OCIUrl: args[1],
Latest: flags.Latest,
Sign: flags.Sign,
CosignKey: flags.CosignKey,
Client: client,
Metadata: meta,
Version: version,
IOStreams: ioStreams,
}

Expand Down Expand Up @@ -261,28 +262,36 @@ func (o *PushModOptions) Run() error {
}

sp.Info("pushing the module...")
digest, err := o.Client.Push(ctx, o.OCIUrl, targetDir, o.Metadata, nil)
idxDigestURL, imgDigestURL, err := o.Client.Push(ctx, o.OCIUrl, o.Version, targetDir, o.Metadata, nil)
if err != nil {
return err
}

// Tag latest version if required
if o.Latest {
if err = o.Client.Tag(ctx, digest, LatestVersion); err != nil {
return fmt.Errorf("tagging module version as latest failed: %w", err)
if err = o.Client.Tag(ctx, imgDigestURL, LatestVersion); err != nil {
return fmt.Errorf("tagging module image version as latest failed: %w", err)
}
if err = o.Client.Tag(ctx, idxDigestURL, LatestVersion); err != nil {
return fmt.Errorf("tagging module index version as latest failed: %w", err)
}
}
sp.Info("pushed successfully\n")
_ = sp.Stop()

// Signs the module with specific provider
if len(o.Sign) != 0 {
err = oci.SignArtifact(o.Sign, digest, o.CosignKey)
err = oci.SignArtifact(o.Sign, imgDigestURL, o.CosignKey)
if err != nil {
return err
}
err = oci.SignArtifact(o.Sign, idxDigestURL, o.CosignKey)
if err != nil {
return err
}
}

sp.Info("pushed successfully\n")
_ = sp.Stop()

return nil
}

Expand All @@ -298,11 +307,16 @@ func (o *PushModOptions) buildModule() (string, error) {
moduleSrc := filepath.Join(o.ModulePath, "src")
goFileSearchPattern := filepath.Join(moduleSrc, "*.go")

// OCIUrl example: oci://ghcr.io/org/my-module-linux_amd64:0.1.0
// prepare platform
if o.Metadata.Platform == nil {
return "", fmt.Errorf("platform is not set in metadata")
}
pOS := o.Metadata.Platform.OS
pArch := o.Metadata.Platform.Architecture

// OCIUrl example: oci://ghcr.io/org/my-module
split := strings.Split(o.OCIUrl, "/")
nameVersion := strings.Split(split[len(split)-1], ":")
name := nameVersion[0]
version := nameVersion[1]
name := split[len(split)-1]

if matches, err := filepath.Glob(goFileSearchPattern); err != nil || len(matches) == 0 {
return "", fmt.Errorf("no go source code files found for 'go build' matching %s", goFileSearchPattern)
Expand All @@ -313,15 +327,9 @@ func (o *PushModOptions) buildModule() (string, error) {
return "", fmt.Errorf("unable to find executable 'go' binary: %w", err)
}

// prepare platform
if o.Metadata.Platform == nil {
return "", fmt.Errorf("platform is not set in metadata")
}
pOS := o.Metadata.Platform.OS
pArch := o.Metadata.Platform.Architecture
output := filepath.Join(targetDir, "_dist", pOS, pArch, "kusion-module-"+name+"_"+version)
output := filepath.Join(targetDir, "_dist", pOS, pArch, "kusion-module-"+name+"_"+o.Version)
if strings.Contains(o.OSArch, "windows") {
output = filepath.Join(targetDir, "_dist", pOS, pArch, "kusion-module-"+name+"_"+version+".exe")
output = filepath.Join(targetDir, "_dist", pOS, pArch, "kusion-module-"+name+"_"+o.Version+".exe")
}

path, err := buildBinary(goBin, pOS, pArch, moduleSrc, output, o.IOStreams)
Expand Down
102 changes: 86 additions & 16 deletions pkg/oci/client/push.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package client

import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
Expand All @@ -11,6 +12,9 @@ import (
"github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/empty"
"github.com/google/go-containerregistry/pkg/v1/mutate"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/google/go-containerregistry/pkg/v1/remote/transport"
"github.com/google/go-containerregistry/pkg/v1/tarball"
"github.com/google/go-containerregistry/pkg/v1/types"

Expand All @@ -25,23 +29,57 @@ const ArtifactTarballFileName = "artifact.tgz"
// - builds tarball from given directory also corresponding layer
// - adds this layer to an empty OpenContainers artifact
// - annotates the artifact with the given annotations
// - uploads the final artifact to the OCI registry
// - check if the target reference exists and if it is an image index. Creates a new image index if it does not exist
// - uploads the final artifact to the OCI registry and appends the artifact to the index,
// - returns the digest URL of the upstream artifact
func (c *Client) Push(ctx context.Context, ociURL, sourceDir string, metadata meta.Metadata, ignorePaths []string) (string, error) {
func (c *Client) Push(
ctx context.Context,
ociURL, version, sourceDir string,
metadata meta.Metadata,
ignorePaths []string,
) (string, string, error) {
ref, err := oci.ParseArtifactRef(ociURL)
if err != nil {
return "", fmt.Errorf("invalid OCI repository url: %w", err)
return "", "", fmt.Errorf("invalid OCI repository url: %w", err)
}

// Check if the target reference exists and if it is an image index
refStr := ref.String()
exists := true
var base v1.ImageIndex

manifest, err := crane.Get(refStr, c.opts.craneOptions...)
if err != nil {
var t *transport.Error
ok := errors.As(err, &t)
if ok && t.StatusCode == 404 {
exists = false
base = empty.Index
} else {
return "", "", fmt.Errorf("get manifest failed: %s, %w", refStr, err)
}
}

if exists {
if !manifest.MediaType.IsIndex() {
return "", "", fmt.Errorf("expected %s to be an index, got %q", refStr, manifest.MediaType)
}
base, err = manifest.ImageIndex()
if err != nil {
return "", "", fmt.Errorf("get manifest image index failed: %s, %w", refStr, err)
}
}

// build image
tmpDir, err := os.MkdirTemp("", "oci")
if err != nil {
return "", err
return "", "", err
}
defer os.RemoveAll(tmpDir)

tmpFile := filepath.Join(tmpDir, ArtifactTarballFileName)
if err := c.Build(tmpFile, sourceDir, ignorePaths); err != nil {
return "", err
if err = c.Build(tmpFile, sourceDir, ignorePaths); err != nil {
return "", "", err
}

// Add missing metadata
Expand All @@ -56,19 +94,19 @@ func (c *Client) Push(ctx context.Context, ociURL, sourceDir string, metadata me

platform := metadata.Platform
if platform == nil {
return "", fmt.Errorf("platform is not set")
return "", "", fmt.Errorf("platform is not set")
}
image, err = mutate.ConfigFile(image, &v1.ConfigFile{
Architecture: platform.Architecture,
OS: platform.OS,
})
if err != nil {
return "", fmt.Errorf("setting image config file failed: %w", err)
return "", "", fmt.Errorf("setting image config file failed: %w", err)
}

layer, err := tarball.LayerFromFile(tmpFile, tarball.WithMediaType(CanonicalContentMediaType))
if err != nil {
return "", fmt.Errorf("creating content layer failed: %w", err)
return "", "", fmt.Errorf("creating content layer failed: %w", err)
}

image, err = mutate.Append(image, mutate.Addendum{
Expand All @@ -78,18 +116,50 @@ func (c *Client) Push(ctx context.Context, ociURL, sourceDir string, metadata me
},
})
if err != nil {
return "", fmt.Errorf("appeding content to artifact failed: %w", err)
return "", "", fmt.Errorf("appeding content to artifact failed: %w", err)
}

imgURL := fmt.Sprintf("%s-%s_%s:%s", ociURL, platform.OS, platform.Architecture, version)
imgRef, err := oci.ParseArtifactRef(imgURL)
if err != nil {
return "", "", fmt.Errorf("invalid image repository url: %w", err)
}
if err = crane.Push(image, imgRef.String(), c.optionsWithContext(ctx)...); err != nil {
return "", "", fmt.Errorf("pushing artifact failed: %w", err)
}
imgDigest, err := image.Digest()
if err != nil {
return "", "", fmt.Errorf("parsing image digest failed: %w", err)
}

if err := crane.Push(image, ref.String(), c.optionsWithContext(ctx)...); err != nil {
return "", fmt.Errorf("pushing artifact failed: %w", err)
cf, err := image.ConfigFile()
if err != nil {
return "", "", fmt.Errorf("parsing image config file failed: %w", err)
}

digest, err := image.Digest()
newDesc, err := partial.Descriptor(image)
if err != nil {
return "", fmt.Errorf("parsing artifact digest failed: %w", err)
return "", "", fmt.Errorf("parsing image descriptor file failed: %w", err)
}
newDesc.Platform = cf.Platform()
addendum := mutate.IndexAddendum{
Add: image,
Descriptor: *newDesc,
}
idx := mutate.AppendManifests(base, addendum)
idxDigest, err := idx.Digest()
if err != nil {
return "", "", fmt.Errorf("parsing index digest failed: %w", err)
}

o := crane.GetOptions(c.opts.craneOptions...)
if err = remote.WriteIndex(ref, idx, o.Remote...); err != nil {
return "", "", fmt.Errorf("pushing image index %s: %w", refStr, err)
}

digestURL := ref.Context().Digest(digest.String()).String()
return fmt.Sprintf("%s%s", oci.OCIRepositoryPrefix, digestURL), nil
idxDigestURL := ref.Context().Digest(idxDigest.String()).String()
imgDigestURL := ref.Context().Digest(imgDigest.String()).String()
idxURL := fmt.Sprintf("%s%s", oci.OCIRepositoryPrefix, idxDigestURL)
imgURL = fmt.Sprintf("%s%s", oci.OCIRepositoryPrefix, imgDigestURL)
return idxURL, imgURL, nil
}

0 comments on commit 41a90a4

Please sign in to comment.