Skip to content

Commit

Permalink
Add command to save images from the cluster
Browse files Browse the repository at this point in the history
This is the opposite command of "minikube image load",
and can be used after doing a "minikube image build".

The default is to save images in the cache, but it is
also possible to save to files or to standard output.
  • Loading branch information
afbjorklund committed Aug 8, 2021
1 parent af84372 commit 386b96f
Show file tree
Hide file tree
Showing 6 changed files with 356 additions and 0 deletions.
74 changes: 74 additions & 0 deletions cmd/minikube/cmd/image.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,77 @@ var loadImageCmd = &cobra.Command{
},
}

func readFile(w io.Writer, tmp string) error {
r, err := os.Open(tmp)
if err != nil {
return err
}
_, err = io.Copy(w, r)
if err != nil {
return err
}
err = r.Close()
if err != nil {
return err
}
return nil
}

// saveImageCmd represents the image load command
var saveImageCmd = &cobra.Command{
Use: "save IMAGE [ARCHIVE | -]",
Short: "Save a image from minikube",
Long: "Save a image from minikube",
Example: "minikube image save image\nminikube image save image image.tar",
Run: func(cmd *cobra.Command, args []string) {
if len(args) == 0 {
exit.Message(reason.Usage, "Please provide an image in the container runtime to save from minikube via <minikube image save IMAGE_NAME>")
}
// Save images from container runtime
profile, err := config.LoadProfile(viper.GetString(config.ProfileName))
if err != nil {
exit.Error(reason.Usage, "loading profile", err)
}

if len(args) > 1 {
output = args[1]

if args[1] == "-" {
tmp, err := ioutil.TempFile("", "image.*.tar")
if err != nil {
exit.Error(reason.GuestImageSave, "Failed to get temp", err)
}
tmp.Close()
output = tmp.Name()
}

if err := machine.DoSaveImages([]string{args[0]}, output, []*config.Profile{profile}, ""); err != nil {
exit.Error(reason.GuestImageSave, "Failed to save image", err)
}

if args[1] == "-" {
err := readFile(os.Stdout, output)
if err != nil {
exit.Error(reason.GuestImageSave, "Failed to read temp", err)
}
os.Remove(output)
}
} else {
if err := machine.SaveAndCacheImages([]string{args[0]}, []*config.Profile{profile}); err != nil {
exit.Error(reason.GuestImageSave, "Failed to save image", err)
}
if imgDaemon || imgRemote {
image.UseDaemon(imgDaemon)
image.UseRemote(imgRemote)
err := image.UploadCachedImage(args[0])
if err != nil {
exit.Error(reason.GuestImageSave, "Failed to save image", err)
}
}
}
},
}

var removeImageCmd = &cobra.Command{
Use: "rm IMAGE [IMAGE...]",
Short: "Remove one or more images",
Expand Down Expand Up @@ -258,5 +329,8 @@ func init() {
buildImageCmd.Flags().StringArrayVar(&buildEnv, "build-env", nil, "Environment variables to pass to the build. (format: key=value)")
buildImageCmd.Flags().StringArrayVar(&buildOpt, "build-opt", nil, "Specify arbitrary flags to pass to the build. (format: key=value)")
imageCmd.AddCommand(buildImageCmd)
saveImageCmd.Flags().BoolVar(&imgDaemon, "daemon", false, "Cache image to docker daemon")
saveImageCmd.Flags().BoolVar(&imgRemote, "remote", false, "Cache image to remote registry")
imageCmd.AddCommand(saveImageCmd)
imageCmd.AddCommand(listImageCmd)
}
58 changes: 58 additions & 0 deletions pkg/minikube/image/image.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,12 @@ import (
"github.com/google/go-containerregistry/pkg/v1/daemon"
"github.com/google/go-containerregistry/pkg/v1/mutate"
"github.com/google/go-containerregistry/pkg/v1/remote"
"github.com/google/go-containerregistry/pkg/v1/tarball"

"github.com/pkg/errors"
"k8s.io/klog/v2"
"k8s.io/minikube/pkg/minikube/constants"
"k8s.io/minikube/pkg/minikube/localpath"
)

const (
Expand Down Expand Up @@ -191,6 +193,62 @@ func retrieveRemote(ref name.Reference, p v1.Platform) (v1.Image, error) {
return img, err
}

// imagePathInCache returns path in local cache directory
func imagePathInCache(img string) string {
f := filepath.Join(constants.ImageCacheDir, img)
f = localpath.SanitizeCacheDir(f)
return f
}

func UploadCachedImage(imgName string) error {
tag, err := name.NewTag(imgName, name.WeakValidation)
if err != nil {
klog.Infof("error parsing image name %s tag %v ", imgName, err)
return err
}
return uploadImage(tag, imgName, imagePathInCache(imgName))
}

func uploadImage(tag name.Tag, imgName string, p string) error {
var err error
var img v1.Image

if !useDaemon && !useRemote {
return fmt.Errorf("neither daemon nor remote")
}

img, err = tarball.ImageFromPath(p, &tag)
if err != nil {
return errors.Wrap(err, "tarball")
}
ref := name.Reference(tag)

klog.Infof("uploading image: %+v from: %s", ref, p)
if useDaemon {
return uploadDaemon(ref, img)
}
if useRemote {
return uploadRemote(ref, img, defaultPlatform)
}
return nil
}

func uploadDaemon(ref name.Reference, img v1.Image) error {
resp, err := daemon.Write(ref, img)
if err != nil {
klog.Warningf("daemon load for %s: %v\n%s", ref, err, resp)
}
return err
}

func uploadRemote(ref name.Reference, img v1.Image, p v1.Platform) error {
err := remote.Write(ref, img, remote.WithAuthFromKeychain(authn.DefaultKeychain), remote.WithPlatform(p))
if err != nil {
klog.Warningf("remote push for %s: %v", ref, err)
}
return err
}

// See https://github.com/kubernetes/minikube/issues/10402
// check if downloaded image Architecture field matches the requested and fix it otherwise
func fixPlatform(ref name.Reference, img v1.Image, p v1.Platform) (v1.Image, error) {
Expand Down
171 changes: 171 additions & 0 deletions pkg/minikube/machine/cache_images.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package machine
import (
"fmt"
"os"
"os/exec"
"path"
"path/filepath"
"sort"
Expand Down Expand Up @@ -48,6 +49,9 @@ var loadRoot = path.Join(vmpath.GuestPersistentDir, "images")
// loadImageLock is used to serialize image loads to avoid overloading the guest VM
var loadImageLock sync.Mutex

// saveRoot is where images should be saved from within the guest VM
var saveRoot = path.Join(vmpath.GuestPersistentDir, "images")

// CacheImagesForBootstrapper will cache images for a bootstrapper
func CacheImagesForBootstrapper(imageRepository string, version string, clusterBootstrapper string) error {
images, err := bootstrapper.GetCachedImageList(imageRepository, version, clusterBootstrapper)
Expand Down Expand Up @@ -309,6 +313,173 @@ func transferAndLoadImage(cr command.Runner, k8s config.KubernetesConfig, src st
return nil
}

// SaveCachedImages saves from the container runtime to the cache
func SaveCachedImages(cc *config.ClusterConfig, runner command.Runner, images []string, cacheDir string) error {
klog.Infof("SaveImages start: %s", images)
start := time.Now()

defer func() {
klog.Infof("SaveImages completed in %s", time.Since(start))
}()

var g errgroup.Group

for _, image := range images {
image := image
g.Go(func() error {
return transferAndSaveCachedImage(runner, cc.KubernetesConfig, image, cacheDir)
})
}
if err := g.Wait(); err != nil {
return errors.Wrap(err, "saving cached images")
}
klog.Infoln("Successfully saved all cached images")
return nil
}

// SaveLocalImages saves images from the container runtime
func SaveLocalImages(cc *config.ClusterConfig, runner command.Runner, images []string, output string) error {
var g errgroup.Group
for _, image := range images {
image := image
g.Go(func() error {
return transferAndSaveImage(runner, cc.KubernetesConfig, output, image)
})
}
if err := g.Wait(); err != nil {
return errors.Wrap(err, "saving images")
}
klog.Infoln("Successfully saved all images")
return nil
}

// SaveAndCacheImages saves images from all profiles into the cache
func SaveAndCacheImages(images []string, profiles []*config.Profile) error {
if len(images) == 0 {
return nil
}

return DoSaveImages(images, "", profiles, constants.ImageCacheDir)
}

// DoSaveImages saves images from all profiles
func DoSaveImages(images []string, output string, profiles []*config.Profile, cacheDir string) error {
api, err := NewAPIClient()
if err != nil {
return errors.Wrap(err, "api")
}
defer api.Close()

klog.Infof("Save images: %q", images)

succeeded := []string{}
failed := []string{}

for _, p := range profiles { // loading images to all running profiles
pName := p.Name // capture the loop variable

c, err := config.Load(pName)
if err != nil {
// Non-fatal because it may race with profile deletion
klog.Errorf("Failed to load profile %q: %v", pName, err)
failed = append(failed, pName)
continue
}

for _, n := range c.Nodes {
m := config.MachineName(*c, n)

status, err := Status(api, m)
if err != nil {
klog.Warningf("error getting status for %s: %v", m, err)
failed = append(failed, m)
continue
}

if status == state.Running.String() { // the not running hosts will load on next start
h, err := api.Load(m)
if err != nil {
klog.Warningf("Failed to load machine %q: %v", m, err)
failed = append(failed, m)
continue
}
cr, err := CommandRunner(h)
if err != nil {
return err
}
if cacheDir != "" {
// saving image names, to cache
err = SaveCachedImages(c, cr, images, cacheDir)
} else {
// saving mage files
err = SaveLocalImages(c, cr, images, output)
}
if err != nil {
failed = append(failed, m)
klog.Warningf("Failed to load cached images for profile %s. make sure the profile is running. %v", pName, err)
continue
}
succeeded = append(succeeded, m)
}
}
}

klog.Infof("succeeded pulling from : %s", strings.Join(succeeded, " "))
klog.Infof("failed pulling from : %s", strings.Join(failed, " "))
// Live pushes are not considered a failure
return nil
}

// transferAndSaveCachedImage transfers and loads a single image from the cache
func transferAndSaveCachedImage(cr command.Runner, k8s config.KubernetesConfig, imgName string, cacheDir string) error {
dst := filepath.Join(cacheDir, imgName)
dst = localpath.SanitizeCacheDir(dst)
return transferAndSaveImage(cr, k8s, dst, imgName)
}

// transferAndSaveImage transfers and loads a single image
func transferAndSaveImage(cr command.Runner, k8s config.KubernetesConfig, dst string, imgName string) error {
r, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime, Runner: cr})
if err != nil {
return errors.Wrap(err, "runtime")
}

klog.Infof("Saving image to: %s", dst)
filename := filepath.Base(dst)

_, err = os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, 0777)
if err != nil {
return err
}

f, err := assets.NewFileAsset(dst, saveRoot, filename, "0644")
if err != nil {
return errors.Wrapf(err, "creating copyable file asset: %s", filename)
}
defer func() {
if err := f.Close(); err != nil {
klog.Warningf("error closing the file %s: %v", f.GetSourcePath(), err)
}
}()

src := path.Join(saveRoot, filename)
args := append([]string{"rm", "-f"}, src)
if _, err := cr.RunCmd(exec.Command("sudo", args...)); err != nil {
return err
}
err = r.SaveImage(imgName, src)
if err != nil {
return errors.Wrapf(err, "%s save %s", r.Name(), src)
}

if err := cr.CopyFrom(f); err != nil {
return errors.Wrap(err, "transferring cached image")
}

klog.Infof("Transferred and saved %s to cache", dst)
return nil
}

// pullImages pulls images to the container run time
func pullImages(cruntime cruntime.Manager, images []string) error {
klog.Infof("PullImages start: %s", images)
Expand Down
2 changes: 2 additions & 0 deletions pkg/minikube/reason/reason.go
Original file line number Diff line number Diff line change
Expand Up @@ -317,6 +317,8 @@ var (
GuestImageRemove = Kind{ID: "GUEST_IMAGE_REMOVE", ExitCode: ExGuestError}
// minikube failed to build an image
GuestImageBuild = Kind{ID: "GUEST_IMAGE_BUILD", ExitCode: ExGuestError}
// minikube failed to push or save an image
GuestImageSave = Kind{ID: "GUEST_IMAGE_SAVE", ExitCode: ExGuestError}
// minikube failed to load host
GuestLoadHost = Kind{ID: "GUEST_LOAD_HOST", ExitCode: ExGuestError}
// minkube failed to create a mount
Expand Down
Loading

0 comments on commit 386b96f

Please sign in to comment.