From 2d80e99cba3d1fb75d2702575aad0c4093ee4472 Mon Sep 17 00:00:00 2001 From: Lavrenti Frobeen Date: Mon, 7 Nov 2022 17:42:24 +0100 Subject: [PATCH] Add support for configurable compression algorithm (gzip, zstd) and compression level MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We want to make the layer compression in kaniko configurable, so we have added two optional command line arguments “--compression” and “--compression-level”. The former allows the user to specify a compression algorithm (zstd, gzip) and the latter can be used to specify the compression level. Depending on the selected compression algorithm and level we modify the set of layerOptions that are used to create tarball layers in `push.go` and `build.go`. The actual implementation of the zstd support can be found in our fork of the go-containerregistry package for which we have filed this PR: google/go-containerregistry#1487 The changes should be fully backwards compatible. --- cmd/executor/cmd/root.go | 4 +++- pkg/config/options.go | 4 ++++ pkg/executor/build.go | 20 ++++++++++++++++---- pkg/executor/push.go | 23 ++++++++++++++++++----- 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/cmd/executor/cmd/root.go b/cmd/executor/cmd/root.go index 4d9c019d54..257e1e8a34 100644 --- a/cmd/executor/cmd/root.go +++ b/cmd/executor/cmd/root.go @@ -225,6 +225,8 @@ func addKanikoOptionsFlags() { RootCmd.PersistentFlags().StringVarP(&opts.ImageNameDigestFile, "image-name-with-digest-file", "", "", "Specify a file to save the image name w/ digest of the built image to.") RootCmd.PersistentFlags().StringVarP(&opts.ImageNameTagDigestFile, "image-name-tag-with-digest-file", "", "", "Specify a file to save the image name w/ image tag w/ digest of the built image to.") RootCmd.PersistentFlags().StringVarP(&opts.OCILayoutPath, "oci-layout-path", "", "", "Path to save the OCI image layout of the built image.") + RootCmd.PersistentFlags().StringVarP(&opts.Compression, "compression", "", "", "Compression algorithm (gzip, zstd)") + RootCmd.PersistentFlags().IntVarP(&opts.CompressionLevel, "compression-level", "", -1, "Compression level") RootCmd.PersistentFlags().BoolVarP(&opts.Cache, "cache", "", false, "Use cache when building image") RootCmd.PersistentFlags().BoolVarP(&opts.CompressedCaching, "compressed-caching", "", true, "Compress the cached layers. Decreases build time, but increases memory usage.") RootCmd.PersistentFlags().BoolVarP(&opts.Cleanup, "cleanup", "", false, "Clean the filesystem at the end") @@ -446,7 +448,7 @@ func exit(err error) { exitWithCode(err, 1) } -//exits with the given error and exit code +// exits with the given error and exit code func exitWithCode(err error, exitCode int) { fmt.Println(err) os.Exit(exitCode) diff --git a/pkg/config/options.go b/pkg/config/options.go index d368db139a..761c3933c3 100644 --- a/pkg/config/options.go +++ b/pkg/config/options.go @@ -22,6 +22,8 @@ import ( "strconv" "strings" "time" + + "github.com/google/go-containerregistry/pkg/compression" ) // CacheOptions are base image cache options that are set by command line arguments @@ -68,6 +70,8 @@ type KanikoOptions struct { ImageNameDigestFile string ImageNameTagDigestFile string OCILayoutPath string + Compression compression.Compression + CompressionLevel int ImageFSExtractRetry int SingleSnapshot bool Reproducible bool diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 2bf379abb7..46f3e57001 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -515,12 +515,24 @@ func (s *stageBuilder) saveSnapshotToLayer(tarPath string) (v1.Layer, error) { return nil, nil } - var layer v1.Layer + var layerOpts []tarball.LayerOption + if s.opts.CompressedCaching == true { - layer, err = tarball.LayerFromFile(tarPath, tarball.WithCompressedCaching) - } else { - layer, err = tarball.LayerFromFile(tarPath) + layerOpts = append(layerOpts, tarball.WithCompressedCaching) + } + + if s.opts.CompressionLevel > 0 { + layerOpts = append(layerOpts, tarball.WithCompressionLevel(s.opts.CompressionLevel)) } + + switch s.opts.Compression { + case "zstd": + layerOpts = append(layerOpts, tarball.WithCompression("zstd")) + default: + // layer already gzipped by default + } + + layer, err := tarball.LayerFromFile(tarPath, layerOpts...) if err != nil { return nil, err } diff --git a/pkg/executor/push.go b/pkg/executor/push.go index 7e2e156a21..1a8b308aa0 100644 --- a/pkg/executor/push.go +++ b/pkg/executor/push.go @@ -296,16 +296,29 @@ func writeImageOutputs(image v1.Image, destRefs []name.Tag) error { // pushLayerToCache pushes layer (tagged with cacheKey) to opts.CacheRepo // if opts.CacheRepo doesn't exist, infer the cache from the given destination func pushLayerToCache(opts *config.KanikoOptions, cacheKey string, tarPath string, createdBy string) error { - var layer v1.Layer - var err error + var layerOpts []tarball.LayerOption if opts.CompressedCaching == true { - layer, err = tarball.LayerFromFile(tarPath, tarball.WithCompressedCaching) - } else { - layer, err = tarball.LayerFromFile(tarPath) + layerOpts = append(layerOpts, tarball.WithCompressedCaching) } + + if opts.CompressionLevel > 0 { + layerOpts = append(layerOpts, tarball.WithCompressionLevel(opts.CompressionLevel)) + } + + switch opts.Compression { + case "zstd": + layerOpts = append(layerOpts, tarball.WithCompression("zstd")) + case "none": + layerOpts = append(layerOpts, tarball.WithCompression("none")) + default: + // layer already gzipped by default + } + + layer, err := tarball.LayerFromFile(tarPath, layerOpts...) if err != nil { return err } + cache, err := cache.Destination(opts, cacheKey) if err != nil { return errors.Wrap(err, "getting cache destination")