Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: group multiple os arch modules by image index #1054

Merged
merged 1 commit into from
Apr 19, 2024
Merged

feat: group multiple os arch modules by image index #1054

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
64 changes: 36 additions & 28 deletions pkg/cmd/mod/mod_push.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ func NewCmdPush(ioStreams genericiooptions.IOStreams) *cobra.Command {
func (flags *PushModFlags) AddFlags(cmd *cobra.Command) {
cmd.Flags().StringVarP(&flags.Version, "version", "v", "", "The version of the module e.g. '1.0.0' or '1.0.0-rc.1'.")
cmd.Flags().StringVar(&flags.OSArch, "os-arch", "", "The os arch of the module e.g. 'darwin/arm64', 'linux/amd64'.")
cmd.Flags().BoolVar(&flags.Latest, "latest", flags.Latest, "Tags the current version as the latest stable module version.")
cmd.Flags().BoolVar(&flags.Latest, "latest", true, "Tags the current version as the latest stable module version.")
cmd.Flags().StringVar(&flags.Credentials, "creds", flags.Credentials,
"The credentials token for the OCI registry in <YOUR_TOKEN> or <YOUR_USERNAME>:<YOUR_TOKEN> format.")
cmd.Flags().StringVar(&flags.Sign, "sign", flags.Sign, "Signs the module with the specified provider.")
Expand All @@ -147,18 +147,11 @@ func (flags *PushModFlags) ToOptions(args []string, ioStreams genericiooptions.I
return nil, fmt.Errorf("path to module and OCI registry url are required")
}

// Prepare metadata
if flags.OSArch == "" {
// set as the current OS arch
flags.OSArch = runtime.GOOS + "/" + runtime.GOARCH
}
osArch := strings.Split(flags.OSArch, "/")

// version
version := flags.Version
if _, err := semver.StrictNewVersion(version); err != nil {
return nil, fmt.Errorf("version is not in semver format: %w", err)
}
fullURL := fmt.Sprintf("%s-%s_%s:%s", args[1], osArch[0], osArch[1], version)

// If creds in <token> format, creds must be base64 encoded
if len(flags.Credentials) != 0 && !strings.Contains(flags.Credentials, ":") {
Expand All @@ -179,6 +172,13 @@ func (flags *PushModFlags) ToOptions(args []string, ioStreams genericiooptions.I
info = gitutil.Get(repoRoot)
}

// os arch
if flags.OSArch == "" {
// set as the current OS arch
flags.OSArch = runtime.GOOS + "/" + runtime.GOARCH
}
osArch := strings.Split(flags.OSArch, "/")

meta := metadata.Metadata{
Created: info.CommitDate,
Source: info.RemoteURL,
Expand All @@ -205,12 +205,13 @@ func (flags *PushModFlags) ToOptions(args []string, ioStreams genericiooptions.I

opt := &PushModOptions{
ModulePath: args[0],
OCIUrl: fullURL,
OCIUrl: args[1],
Latest: flags.Latest,
Sign: flags.Sign,
CosignKey: flags.CosignKey,
Client: client,
Metadata: meta,
Version: version,
IOStreams: ioStreams,
}

Expand Down Expand Up @@ -261,28 +262,36 @@ func (o *PushModOptions) Run() error {
}

sp.Info("pushing the module...")
digest, err := o.Client.Push(ctx, o.OCIUrl, targetDir, o.Metadata, nil)
idxDigestURL, imgDigestURL, err := o.Client.Push(ctx, o.OCIUrl, o.Version, targetDir, o.Metadata, nil)
if err != nil {
return err
}

// Tag latest version if required
if o.Latest {
if err = o.Client.Tag(ctx, digest, LatestVersion); err != nil {
return fmt.Errorf("tagging module version as latest failed: %w", err)
if err = o.Client.Tag(ctx, imgDigestURL, LatestVersion); err != nil {
return fmt.Errorf("tagging module image version as latest failed: %w", err)
}
if err = o.Client.Tag(ctx, idxDigestURL, LatestVersion); err != nil {
return fmt.Errorf("tagging module index version as latest failed: %w", err)
}
}
sp.Info("pushed successfully\n")
_ = sp.Stop()

// Signs the module with specific provider
if len(o.Sign) != 0 {
err = oci.SignArtifact(o.Sign, digest, o.CosignKey)
err = oci.SignArtifact(o.Sign, imgDigestURL, o.CosignKey)
if err != nil {
return err
}
err = oci.SignArtifact(o.Sign, idxDigestURL, o.CosignKey)
if err != nil {
return err
}
}

sp.Info("pushed successfully\n")
_ = sp.Stop()

return nil
}

Expand All @@ -298,11 +307,16 @@ func (o *PushModOptions) buildModule() (string, error) {
moduleSrc := filepath.Join(o.ModulePath, "src")
goFileSearchPattern := filepath.Join(moduleSrc, "*.go")

// OCIUrl example: oci://ghcr.io/org/my-module-linux_amd64:0.1.0
// prepare platform
if o.Metadata.Platform == nil {
return "", fmt.Errorf("platform is not set in metadata")
}
pOS := o.Metadata.Platform.OS
pArch := o.Metadata.Platform.Architecture

// OCIUrl example: oci://ghcr.io/org/my-module
split := strings.Split(o.OCIUrl, "/")
nameVersion := strings.Split(split[len(split)-1], ":")
name := nameVersion[0]
version := nameVersion[1]
name := split[len(split)-1]

if matches, err := filepath.Glob(goFileSearchPattern); err != nil || len(matches) == 0 {
return "", fmt.Errorf("no go source code files found for 'go build' matching %s", goFileSearchPattern)
Expand All @@ -313,15 +327,9 @@ func (o *PushModOptions) buildModule() (string, error) {
return "", fmt.Errorf("unable to find executable 'go' binary: %w", err)
}

// prepare platform
if o.Metadata.Platform == nil {
return "", fmt.Errorf("platform is not set in metadata")
}
pOS := o.Metadata.Platform.OS
pArch := o.Metadata.Platform.Architecture
output := filepath.Join(targetDir, "_dist", pOS, pArch, "kusion-module-"+name+"_"+version)
output := filepath.Join(targetDir, "_dist", pOS, pArch, "kusion-module-"+name+"_"+o.Version)
if strings.Contains(o.OSArch, "windows") {
output = filepath.Join(targetDir, "_dist", pOS, pArch, "kusion-module-"+name+"_"+version+".exe")
output = filepath.Join(targetDir, "_dist", pOS, pArch, "kusion-module-"+name+"_"+o.Version+".exe")
}

path, err := buildBinary(goBin, pOS, pArch, moduleSrc, output, o.IOStreams)
Expand Down
102 changes: 86 additions & 16 deletions pkg/oci/client/push.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package client

import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
Expand All @@ -11,6 +12,9 @@ import (
"github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/empty"
"github.com/google/go-containerregistry/pkg/v1/mutate"
"github.com/google/go-containerregistry/pkg/v1/partial"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/google/go-containerregistry/pkg/v1/remote/transport"
"github.com/google/go-containerregistry/pkg/v1/tarball"
"github.com/google/go-containerregistry/pkg/v1/types"

Expand All @@ -25,23 +29,57 @@ const ArtifactTarballFileName = "artifact.tgz"
// - builds tarball from given directory also corresponding layer
// - adds this layer to an empty OpenContainers artifact
// - annotates the artifact with the given annotations
// - uploads the final artifact to the OCI registry
// - check if the target reference exists and if it is an image index. Creates a new image index if it does not exist
// - uploads the final artifact to the OCI registry and appends the artifact to the index,
// - returns the digest URL of the upstream artifact
func (c *Client) Push(ctx context.Context, ociURL, sourceDir string, metadata meta.Metadata, ignorePaths []string) (string, error) {
func (c *Client) Push(
ctx context.Context,
ociURL, version, sourceDir string,
metadata meta.Metadata,
ignorePaths []string,
) (string, string, error) {
ref, err := oci.ParseArtifactRef(ociURL)
if err != nil {
return "", fmt.Errorf("invalid OCI repository url: %w", err)
return "", "", fmt.Errorf("invalid OCI repository url: %w", err)
}

// Check if the target reference exists and if it is an image index
refStr := ref.String()
exists := true
var base v1.ImageIndex

manifest, err := crane.Get(refStr, c.opts.craneOptions...)
if err != nil {
var t *transport.Error
ok := errors.As(err, &t)
if ok && t.StatusCode == 404 {
exists = false
base = empty.Index
} else {
return "", "", fmt.Errorf("get manifest failed: %s, %w", refStr, err)
}
}

if exists {
if !manifest.MediaType.IsIndex() {
return "", "", fmt.Errorf("expected %s to be an index, got %q", refStr, manifest.MediaType)
}
base, err = manifest.ImageIndex()
if err != nil {
return "", "", fmt.Errorf("get manifest image index failed: %s, %w", refStr, err)
}
}

// build image
tmpDir, err := os.MkdirTemp("", "oci")
if err != nil {
return "", err
return "", "", err
}
defer os.RemoveAll(tmpDir)

tmpFile := filepath.Join(tmpDir, ArtifactTarballFileName)
if err := c.Build(tmpFile, sourceDir, ignorePaths); err != nil {
return "", err
if err = c.Build(tmpFile, sourceDir, ignorePaths); err != nil {
return "", "", err
}

// Add missing metadata
Expand All @@ -56,19 +94,19 @@ func (c *Client) Push(ctx context.Context, ociURL, sourceDir string, metadata me

platform := metadata.Platform
if platform == nil {
return "", fmt.Errorf("platform is not set")
return "", "", fmt.Errorf("platform is not set")
}
image, err = mutate.ConfigFile(image, &v1.ConfigFile{
Architecture: platform.Architecture,
OS: platform.OS,
})
if err != nil {
return "", fmt.Errorf("setting image config file failed: %w", err)
return "", "", fmt.Errorf("setting image config file failed: %w", err)
}

layer, err := tarball.LayerFromFile(tmpFile, tarball.WithMediaType(CanonicalContentMediaType))
if err != nil {
return "", fmt.Errorf("creating content layer failed: %w", err)
return "", "", fmt.Errorf("creating content layer failed: %w", err)
}

image, err = mutate.Append(image, mutate.Addendum{
Expand All @@ -78,18 +116,50 @@ func (c *Client) Push(ctx context.Context, ociURL, sourceDir string, metadata me
},
})
if err != nil {
return "", fmt.Errorf("appeding content to artifact failed: %w", err)
return "", "", fmt.Errorf("appeding content to artifact failed: %w", err)
}

imgURL := fmt.Sprintf("%s-%s_%s:%s", ociURL, platform.OS, platform.Architecture, version)
imgRef, err := oci.ParseArtifactRef(imgURL)
if err != nil {
return "", "", fmt.Errorf("invalid image repository url: %w", err)
}
if err = crane.Push(image, imgRef.String(), c.optionsWithContext(ctx)...); err != nil {
return "", "", fmt.Errorf("pushing artifact failed: %w", err)
}
imgDigest, err := image.Digest()
if err != nil {
return "", "", fmt.Errorf("parsing image digest failed: %w", err)
}

if err := crane.Push(image, ref.String(), c.optionsWithContext(ctx)...); err != nil {
return "", fmt.Errorf("pushing artifact failed: %w", err)
cf, err := image.ConfigFile()
if err != nil {
return "", "", fmt.Errorf("parsing image config file failed: %w", err)
}

digest, err := image.Digest()
newDesc, err := partial.Descriptor(image)
if err != nil {
return "", fmt.Errorf("parsing artifact digest failed: %w", err)
return "", "", fmt.Errorf("parsing image descriptor file failed: %w", err)
}
newDesc.Platform = cf.Platform()
addendum := mutate.IndexAddendum{
Add: image,
Descriptor: *newDesc,
}
idx := mutate.AppendManifests(base, addendum)
idxDigest, err := idx.Digest()
if err != nil {
return "", "", fmt.Errorf("parsing index digest failed: %w", err)
}

o := crane.GetOptions(c.opts.craneOptions...)
if err = remote.WriteIndex(ref, idx, o.Remote...); err != nil {
return "", "", fmt.Errorf("pushing image index %s: %w", refStr, err)
}

digestURL := ref.Context().Digest(digest.String()).String()
return fmt.Sprintf("%s%s", oci.OCIRepositoryPrefix, digestURL), nil
idxDigestURL := ref.Context().Digest(idxDigest.String()).String()
imgDigestURL := ref.Context().Digest(imgDigest.String()).String()
idxURL := fmt.Sprintf("%s%s", oci.OCIRepositoryPrefix, idxDigestURL)
imgURL = fmt.Sprintf("%s%s", oci.OCIRepositoryPrefix, imgDigestURL)
return idxURL, imgURL, nil
}