Skip to content

Commit

Permalink
add caching copy layers back (#1518)
Browse files Browse the repository at this point in the history
* add caching copy layers back

* fix test

* lint

* fix test - 2

* Add integration test

* fix lint
  • Loading branch information
tejal29 authored Dec 11, 2020
1 parent dde98a8 commit b04399e
Show file tree
Hide file tree
Showing 11 changed files with 405 additions and 16 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -386,7 +386,7 @@ as a remote image destination:
### Caching

#### Caching Layers
kaniko can cache layers created by `RUN` commands in a remote repository.
kaniko can cache layers created by `RUN` and `Copy` (configured by flag `--cache-copy-layers`) commands in a remote repository.
Before executing a command, kaniko checks the cache for the layer.
If it exists, kaniko will pull and extract the cached layer instead of executing the command.
If not, kaniko will execute the command and then push the newly created layer to the cache.
Expand Down
1 change: 1 addition & 0 deletions cmd/executor/cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,7 @@ func addKanikoOptionsFlags() {
RootCmd.PersistentFlags().BoolVarP(&opts.SkipUnusedStages, "skip-unused-stages", "", false, "Build only used stages if defined to true. Otherwise it builds by default all stages, even the unnecessaries ones until it reaches the target stage / end of Dockerfile")
RootCmd.PersistentFlags().BoolVarP(&opts.RunV2, "use-new-run", "", false, "Use the experimental run implementation for detecting changes without requiring file system snapshots.")
RootCmd.PersistentFlags().Var(&opts.Git, "git", "Branch to clone if build context is a git repository")
RootCmd.PersistentFlags().BoolVarP(&opts.CacheCopyLayers, "cache-copy-layers", "", false, "Caches copy layers")
}

// addHiddenFlags marks certain flags as hidden from the executor help text
Expand Down
5 changes: 4 additions & 1 deletion integration/images.go
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ func populateVolumeCache() error {
}

// buildCachedImages builds the images for testing caching via kaniko where version is the nth time this image has been built
func (d *DockerFileBuilder) buildCachedImages(config *integrationTestConfig, cacheRepo, dockerfilesPath string, version int) error {
func (d *DockerFileBuilder) buildCachedImages(config *integrationTestConfig, cacheRepo, dockerfilesPath string, version int, args []string) error {
imageRepo, serviceAccount := config.imageRepo, config.serviceAccount
_, ex, _, _ := runtime.Caller(0)
cwd := filepath.Dir(ex)
Expand All @@ -334,6 +334,9 @@ func (d *DockerFileBuilder) buildCachedImages(config *integrationTestConfig, cac
cacheFlag,
"--cache-repo", cacheRepo,
"--cache-dir", cacheDir)
for _, v := range args {
dockerRunFlags = append(dockerRunFlags, v)
}
kanikoCmd := exec.Command("docker", dockerRunFlags...)

_, err := RunCommandWithoutTest(kanikoCmd)
Expand Down
8 changes: 6 additions & 2 deletions integration/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -464,17 +464,21 @@ func buildImage(t *testing.T, dockerfile string, imageBuilder *DockerFileBuilder
func TestCache(t *testing.T) {
populateVolumeCache()
for dockerfile := range imageBuilder.TestCacheDockerfiles {
args := []string{}
if dockerfile == "Dockerfile_test_cache_copy" {
args = append(args, "--cache-copy-layers=true")
}
t.Run("test_cache_"+dockerfile, func(t *testing.T) {
dockerfile := dockerfile
t.Parallel()

cache := filepath.Join(config.imageRepo, "cache", fmt.Sprintf("%v", time.Now().UnixNano()))
// Build the initial image which will cache layers
if err := imageBuilder.buildCachedImages(config, cache, dockerfilesPath, 0); err != nil {
if err := imageBuilder.buildCachedImages(config, cache, dockerfilesPath, 0, args); err != nil {
t.Fatalf("error building cached image for the first time: %v", err)
}
// Build the second image which should pull from the cache
if err := imageBuilder.buildCachedImages(config, cache, dockerfilesPath, 1); err != nil {
if err := imageBuilder.buildCachedImages(config, cache, dockerfilesPath, 1, args); err != nil {
t.Fatalf("error building cached image for the first time: %v", err)
}
// Make sure both images are the same
Expand Down
4 changes: 2 additions & 2 deletions pkg/commands/commands.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,15 +60,15 @@ type DockerCommand interface {
ShouldDetectDeletedFiles() bool
}

func GetCommand(cmd instructions.Command, fileContext util.FileContext, useNewRun bool) (DockerCommand, error) {
func GetCommand(cmd instructions.Command, fileContext util.FileContext, useNewRun bool, cacheCopy bool) (DockerCommand, error) {
switch c := cmd.(type) {
case *instructions.RunCommand:
if useNewRun {
return &RunMarkerCommand{cmd: c}, nil
}
return &RunCommand{cmd: c}, nil
case *instructions.CopyCommand:
return &CopyCommand{cmd: c, fileContext: fileContext}, nil
return &CopyCommand{cmd: c, fileContext: fileContext, shdCache: cacheCopy}, nil
case *instructions.ExposeCommand:
return &ExposeCommand{cmd: c}, nil
case *instructions.EnvCommand:
Expand Down
81 changes: 81 additions & 0 deletions pkg/commands/copy.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package commands

import (
"fmt"
"os"
"path/filepath"
"strings"
Expand All @@ -41,6 +42,7 @@ type CopyCommand struct {
cmd *instructions.CopyCommand
fileContext util.FileContext
snapshotFiles []string
shdCache bool
}

func (c *CopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error {
Expand Down Expand Up @@ -145,6 +147,85 @@ func (c *CopyCommand) From() string {
return c.cmd.From
}

func (c *CopyCommand) ShouldCacheOutput() bool {
return c.shdCache
}

// CacheCommand returns true since this command should be cached
func (c *CopyCommand) CacheCommand(img v1.Image) DockerCommand {
return &CachingCopyCommand{
img: img,
cmd: c.cmd,
fileContext: c.fileContext,
extractFn: util.ExtractFile,
}
}

type CachingCopyCommand struct {
BaseCommand
caching
img v1.Image
extractedFiles []string
cmd *instructions.CopyCommand
fileContext util.FileContext
extractFn util.ExtractFunction
}

func (cr *CachingCopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error {
logrus.Infof("Found cached layer, extracting to filesystem")
var err error

if cr.img == nil {
return errors.New(fmt.Sprintf("cached command image is nil %v", cr.String()))
}

layers, err := cr.img.Layers()
if err != nil {
return errors.Wrapf(err, "retrieve image layers")
}

if len(layers) != 1 {
return errors.New(fmt.Sprintf("expected %d layers but got %d", 1, len(layers)))
}

cr.layer = layers[0]
cr.extractedFiles, err = util.GetFSFromLayers(kConfig.RootDir, layers, util.ExtractFunc(cr.extractFn), util.IncludeWhiteout())

logrus.Debugf("extractedFiles: %s", cr.extractedFiles)
if err != nil {
return errors.Wrap(err, "extracting fs from image")
}

return nil
}

func (cr *CachingCopyCommand) FilesUsedFromContext(config *v1.Config, buildArgs *dockerfile.BuildArgs) ([]string, error) {
return copyCmdFilesUsedFromContext(config, buildArgs, cr.cmd, cr.fileContext)
}

func (cr *CachingCopyCommand) FilesToSnapshot() []string {
f := cr.extractedFiles
logrus.Debugf("%d files extracted by caching copy command", len(f))
logrus.Tracef("Extracted files: %s", f)

return f
}

func (cr *CachingCopyCommand) MetadataOnly() bool {
return false
}

func (cr *CachingCopyCommand) String() string {
if cr.cmd == nil {
return "nil command"
}
return cr.cmd.String()
}

func (cr *CachingCopyCommand) From() string {
return cr.cmd.From
}

func resolveIfSymlink(destPath string) (string, error) {
if !filepath.IsAbs(destPath) {
return "", errors.New("dest path must be abs")
Expand Down
155 changes: 155 additions & 0 deletions pkg/commands/copy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ limitations under the License.
package commands

import (
"archive/tar"
"fmt"
"io"
"io/ioutil"
Expand Down Expand Up @@ -105,6 +106,160 @@ func setupTestTemp() string {

return tempDir
}

func Test_CachingCopyCommand_ExecuteCommand(t *testing.T) {
tempDir := setupTestTemp()

tarContent, err := prepareTarFixture([]string{"foo.txt"})
if err != nil {
t.Errorf("couldn't prepare tar fixture %v", err)
}

config := &v1.Config{}
buildArgs := &dockerfile.BuildArgs{}

type testCase struct {
desctiption string
expectLayer bool
expectErr bool
count *int
expectedCount int
command *CachingCopyCommand
extractedFiles []string
contextFiles []string
}
testCases := []testCase{
func() testCase {
err = ioutil.WriteFile(filepath.Join(tempDir, "foo.txt"), []byte("meow"), 0644)
if err != nil {
t.Errorf("couldn't write tempfile %v", err)
t.FailNow()
}

c := &CachingCopyCommand{
img: fakeImage{
ImageLayers: []v1.Layer{
fakeLayer{TarContent: tarContent},
},
},
fileContext: util.FileContext{Root: tempDir},
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{
"foo.txt", "foo.txt",
},
},
}
count := 0
tc := testCase{
desctiption: "with valid image and valid layer",
count: &count,
expectedCount: 1,
expectLayer: true,
extractedFiles: []string{"/foo.txt"},
contextFiles: []string{"foo.txt"},
}
c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error {
*tc.count++
return nil
}
tc.command = c
return tc
}(),
func() testCase {
c := &CachingCopyCommand{}
tc := testCase{
desctiption: "with no image",
expectErr: true,
}
c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error {
return nil
}
tc.command = c
return tc
}(),
func() testCase {
c := &CachingCopyCommand{
img: fakeImage{},
}
c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error {
return nil
}
return testCase{
desctiption: "with image containing no layers",
expectErr: true,
command: c,
}
}(),
func() testCase {
c := &CachingCopyCommand{
img: fakeImage{
ImageLayers: []v1.Layer{
fakeLayer{},
},
},
}
c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error {
return nil
}
tc := testCase{
desctiption: "with image one layer which has no tar content",
expectErr: false, // this one probably should fail but doesn't because of how ExecuteCommand and util.GetFSFromLayers are implemented - cvgw- 2019-11-25
expectLayer: true,
}
tc.command = c
return tc
}(),
}

for _, tc := range testCases {
t.Run(tc.desctiption, func(t *testing.T) {
c := tc.command
err := c.ExecuteCommand(config, buildArgs)
if !tc.expectErr && err != nil {
t.Errorf("Expected err to be nil but was %v", err)
} else if tc.expectErr && err == nil {
t.Error("Expected err but was nil")
}

if tc.count != nil {
if *tc.count != tc.expectedCount {
t.Errorf("Expected extractFn to be called %v times but was called %v times", tc.expectedCount, *tc.count)
}
for _, file := range tc.extractedFiles {
match := false
cFiles := c.FilesToSnapshot()
for _, cFile := range cFiles {
if file == cFile {
match = true
break
}
}
if !match {
t.Errorf("Expected extracted files to include %v but did not %v", file, cFiles)
}
}

cmdFiles, err := c.FilesUsedFromContext(
config, buildArgs,
)
if err != nil {
t.Errorf("failed to get files used from context from command %v", err)
}

if len(cmdFiles) != len(tc.contextFiles) {
t.Errorf("expected files used from context to equal %v but was %v", tc.contextFiles, cmdFiles)
}
}

if c.layer == nil && tc.expectLayer {
t.Error("expected the command to have a layer set but instead was nil")
} else if c.layer != nil && !tc.expectLayer {
t.Error("expected the command to have no layer set but instead found a layer")
}
})
}
}

func TestCopyExecuteCmd(t *testing.T) {
tempDir := setupTestTemp()
defer os.RemoveAll(tempDir)
Expand Down
1 change: 1 addition & 0 deletions pkg/config/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ type KanikoOptions struct {
IgnoreVarRun bool
SkipUnusedStages bool
RunV2 bool
CacheCopyLayers bool
Git KanikoGitOptions
}

Expand Down
3 changes: 2 additions & 1 deletion pkg/executor/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ func newStageBuilder(opts *config.KanikoOptions, stage config.KanikoStage, cross
}

for _, cmd := range s.stage.Commands {
command, err := commands.GetCommand(cmd, fileContext, opts.RunV2)
command, err := commands.GetCommand(cmd, fileContext, opts.RunV2, opts.CacheCopyLayers)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -184,6 +184,7 @@ func (s *stageBuilder) populateCompositeKey(command fmt.Stringer, files []string
compositeKey.AddKey(resolvedCmd)
switch v := command.(type) {
case *commands.CopyCommand:
case *commands.CachingCopyCommand:
compositeKey = s.populateCopyCmdCompositeKey(command, v.From(), compositeKey)
}

Expand Down
Loading

0 comments on commit b04399e

Please sign in to comment.