diff --git a/pkg/executor/build.go b/pkg/executor/build.go index aadd797115..b8e7f85b89 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -156,8 +156,10 @@ func (s *stageBuilder) populateCompositeKey(command fmt.Stringer, files []string compositeKey = s.populateCopyCmdCompositeKey(command, v.From(), compositeKey) } + srcCtx := s.opts.SrcContext + for _, f := range files { - if err := compositeKey.AddPath(f); err != nil { + if err := compositeKey.AddPath(f, srcCtx); err != nil { return compositeKey, err } } diff --git a/pkg/executor/build_test.go b/pkg/executor/build_test.go index db3f551548..64046bc293 100644 --- a/pkg/executor/build_test.go +++ b/pkg/executor/build_test.go @@ -538,7 +538,7 @@ func Test_stageBuilder_build(t *testing.T) { filePath := filepath.Join(dir, file) ch := NewCompositeCache("", "meow") - ch.AddPath(filePath) + ch.AddPath(filePath, "") hash, err := ch.Hash() if err != nil { t.Errorf("couldn't create hash %v", err) @@ -570,7 +570,7 @@ func Test_stageBuilder_build(t *testing.T) { filePath := filepath.Join(dir, file) ch := NewCompositeCache("", "meow") - ch.AddPath(filePath) + ch.AddPath(filePath, "") hash, err := ch.Hash() if err != nil { t.Errorf("couldn't create hash %v", err) @@ -618,7 +618,7 @@ func Test_stageBuilder_build(t *testing.T) { tarContent := generateTar(t, dir, filename) ch := NewCompositeCache("", "") - ch.AddPath(filepath) + ch.AddPath(filepath, "") hash, err := ch.Hash() if err != nil { @@ -662,7 +662,7 @@ func Test_stageBuilder_build(t *testing.T) { } filePath := filepath.Join(dir, filename) ch := NewCompositeCache("", "") - ch.AddPath(filePath) + ch.AddPath(filePath, "") hash, err := ch.Hash() if err != nil { @@ -713,7 +713,7 @@ func Test_stageBuilder_build(t *testing.T) { } ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename)) - ch.AddPath(filePath) + ch.AddPath(filePath, "") hash2, err := ch.Hash() if err != nil { @@ -721,7 +721,7 @@ func Test_stageBuilder_build(t *testing.T) { } ch = NewCompositeCache("", fmt.Sprintf("COPY %s foo.txt", filename)) ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename)) - ch.AddPath(filePath) + ch.AddPath(filePath, "") image := fakeImage{ ImageLayers: []v1.Layer{ @@ -777,14 +777,14 @@ COPY %s bar.txt } filePath := filepath.Join(dir, filename) ch := NewCompositeCache("", fmt.Sprintf("COPY %s foo.txt", filename)) - ch.AddPath(filePath) + ch.AddPath(filePath, "") hash1, err := ch.Hash() if err != nil { t.Errorf("couldn't create hash %v", err) } ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename)) - ch.AddPath(filePath) + ch.AddPath(filePath, "") hash2, err := ch.Hash() if err != nil { @@ -792,7 +792,7 @@ COPY %s bar.txt } ch = NewCompositeCache("", fmt.Sprintf("COPY %s foo.txt", filename)) ch.AddKey(fmt.Sprintf("COPY %s bar.txt", filename)) - ch.AddPath(filePath) + ch.AddPath(filePath, "") image := fakeImage{ ImageLayers: []v1.Layer{ diff --git a/pkg/executor/composite_cache.go b/pkg/executor/composite_cache.go index ae8ff06099..df1eb5ef85 100644 --- a/pkg/executor/composite_cache.go +++ b/pkg/executor/composite_cache.go @@ -54,18 +54,28 @@ func (s *CompositeCache) Hash() (string, error) { return util.SHA256(strings.NewReader(s.Key())) } -func (s *CompositeCache) AddPath(p string) error { +func (s *CompositeCache) AddPath(p, context string) error { sha := sha256.New() fi, err := os.Lstat(p) if err != nil { return err } + if fi.Mode().IsDir() { - k, err := HashDir(p) + empty, k, err := hashDir(p, context) if err != nil { return err } - s.keys = append(s.keys, k) + + // Only add the hash of this directory to the key + // if there is any whitelisted content. + if !empty || !util.ExcludeFile(p, context) { + s.keys = append(s.keys, k) + } + return nil + } + + if util.ExcludeFile(p, context) { return nil } fh, err := util.CacheHasher()(p) @@ -81,12 +91,18 @@ func (s *CompositeCache) AddPath(p string) error { } // HashDir returns a hash of the directory. -func HashDir(p string) (string, error) { +func hashDir(p, context string) (bool, string, error) { sha := sha256.New() + empty := true if err := filepath.Walk(p, func(path string, fi os.FileInfo, err error) error { if err != nil { return err } + exclude := util.ExcludeFile(path, context) + if exclude { + return nil + } + fileHash, err := util.CacheHasher()(path) if err != nil { return err @@ -94,10 +110,11 @@ func HashDir(p string) (string, error) { if _, err := sha.Write([]byte(fileHash)); err != nil { return err } + empty = false return nil }); err != nil { - return "", err + return false, "", err } - return fmt.Sprintf("%x", sha.Sum(nil)), nil + return empty, fmt.Sprintf("%x", sha.Sum(nil)), nil } diff --git a/pkg/executor/composite_cache_test.go b/pkg/executor/composite_cache_test.go index a3c7f55af6..764364e93e 100644 --- a/pkg/executor/composite_cache_test.go +++ b/pkg/executor/composite_cache_test.go @@ -19,9 +19,12 @@ package executor import ( "io/ioutil" "os" + "path" "path/filepath" "reflect" "testing" + + "github.com/GoogleContainerTools/kaniko/pkg/util" ) func Test_NewCompositeCache(t *testing.T) { @@ -77,7 +80,7 @@ func Test_CompositeCache_AddPath_dir(t *testing.T) { fn := func() string { r := NewCompositeCache() - if err := r.AddPath(tmpDir); err != nil { + if err := r.AddPath(tmpDir, ""); err != nil { t.Errorf("expected error to be nil but was %v", err) } @@ -115,7 +118,7 @@ func Test_CompositeCache_AddPath_file(t *testing.T) { p := tmpfile.Name() fn := func() string { r := NewCompositeCache() - if err := r.AddPath(p); err != nil { + if err := r.AddPath(p, ""); err != nil { t.Errorf("expected error to be nil but was %v", err) } @@ -135,3 +138,433 @@ func Test_CompositeCache_AddPath_file(t *testing.T) { t.Errorf("expected hash %v to equal hash %v", hash1, hash2) } } + +func createFilesystemStructure(root string, directories, files []string) error { + for _, d := range directories { + dirPath := path.Join(root, d) + if err := os.MkdirAll(dirPath, 0755); err != nil { + return err + } + } + + for _, fileName := range files { + filePath := path.Join(root, fileName) + err := ioutil.WriteFile(filePath, []byte(fileName), 0644) + if err != nil { + return err + } + } + + return nil +} + +func setIgnoreContext(content string) error { + dockerIgnoreDir, err := ioutil.TempDir("", "") + if err != nil { + return err + } + defer os.RemoveAll(dockerIgnoreDir) + err = ioutil.WriteFile(dockerIgnoreDir+".dockerignore", []byte(content), 0644) + if err != nil { + return err + } + err = util.GetExcludedFiles(dockerIgnoreDir, "") + if err != nil { + return err + } + return nil +} + +func hashDirectory(dirpath string) (string, error) { + cache1 := NewCompositeCache() + err := cache1.AddPath(dirpath, dirpath) + if err != nil { + return "", err + } + + hash, err := cache1.Hash() + if err != nil { + return "", err + } + return hash, nil +} + +func Test_CompositeKey_AddPath_Works(t *testing.T) { + tests := []struct { + name string + directories []string + files []string + }{ + { + name: "empty", + directories: []string{}, + files: []string{}, + }, + { + name: "dirs", + directories: []string{"foo", "bar", "foobar", "f/o/o"}, + files: []string{}, + }, + { + name: "files", + directories: []string{}, + files: []string{"foo", "bar", "foobar"}, + }, + { + name: "all", + directories: []string{"foo", "bar"}, + files: []string{"foo/bar", "bar/baz", "foobar"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testDir1, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Error creating tempdir: %s", err) + } + defer os.RemoveAll(testDir1) + err = createFilesystemStructure(testDir1, test.directories, test.files) + if err != nil { + t.Fatalf("Error creating filesytem structure: %s", err) + } + + testDir2, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Error creating tempdir: %s", err) + } + defer os.RemoveAll(testDir2) + err = createFilesystemStructure(testDir2, test.directories, test.files) + if err != nil { + t.Fatalf("Error creating filesytem structure: %s", err) + } + + hash1, err := hashDirectory(testDir1) + if err != nil { + t.Fatalf("Failed to calculate hash: %s", err) + } + hash2, err := hashDirectory(testDir2) + if err != nil { + t.Fatalf("Failed to calculate hash: %s", err) + } + + if hash1 != hash2 { + t.Errorf("Expected equal hashes, got: %s and %s", hash1, hash2) + } + }) + } +} + +func Test_CompositeKey_AddPath_WithExtraFile_Works(t *testing.T) { + tests := []struct { + name string + directories []string + files []string + extraFile string + }{ + { + name: "empty", + directories: []string{}, + files: []string{}, + extraFile: "file", + }, + { + name: "dirs", + directories: []string{"foo", "bar", "foobar", "f/o/o"}, + files: []string{}, + extraFile: "f/o/o/extra", + }, + { + name: "files", + directories: []string{}, + files: []string{"foo", "bar", "foobar"}, + extraFile: "foo.extra", + }, + { + name: "all", + directories: []string{"foo", "bar"}, + files: []string{"foo/bar", "bar/baz", "foobar"}, + extraFile: "bar/extra", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testDir1, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Error creating tempdir: %s", err) + } + defer os.RemoveAll(testDir1) + err = createFilesystemStructure(testDir1, test.directories, test.files) + if err != nil { + t.Fatalf("Error creating filesytem structure: %s", err) + } + + testDir2, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Error creating tempdir: %s", err) + } + defer os.RemoveAll(testDir2) + err = createFilesystemStructure(testDir2, test.directories, test.files) + if err != nil { + t.Fatalf("Error creating filesytem structure: %s", err) + } + extraPath := path.Join(testDir2, test.extraFile) + err = ioutil.WriteFile(extraPath, []byte(test.extraFile), 0644) + if err != nil { + t.Fatalf("Error creating filesytem structure: %s", err) + } + + hash1, err := hashDirectory(testDir1) + if err != nil { + t.Fatalf("Failed to calculate hash: %s", err) + } + hash2, err := hashDirectory(testDir2) + if err != nil { + t.Fatalf("Failed to calculate hash: %s", err) + } + + if hash1 == hash2 { + t.Errorf("Expected different hashes, got: %s and %s", hash1, hash2) + } + }) + } +} + +func Test_CompositeKey_AddPath_WithExtraDir_Works(t *testing.T) { + tests := []struct { + name string + directories []string + files []string + extraDir string + }{ + { + name: "empty", + directories: []string{}, + files: []string{}, + extraDir: "extra", + }, + { + name: "dirs", + directories: []string{"foo", "bar", "foobar", "f/o/o"}, + files: []string{}, + extraDir: "f/o/o/extra", + }, + { + name: "files", + directories: []string{}, + files: []string{"foo", "bar", "foobar"}, + extraDir: "foo.extra", + }, + { + name: "all", + directories: []string{"foo", "bar"}, + files: []string{"foo/bar", "bar/baz", "foobar"}, + extraDir: "bar/extra", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testDir1, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Error creating tempdir: %s", err) + } + defer os.RemoveAll(testDir1) + err = createFilesystemStructure(testDir1, test.directories, test.files) + if err != nil { + t.Fatalf("Error creating filesytem structure: %s", err) + } + + testDir2, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Error creating tempdir: %s", err) + } + defer os.RemoveAll(testDir2) + err = createFilesystemStructure(testDir2, test.directories, test.files) + if err != nil { + t.Fatalf("Error creating filesytem structure: %s", err) + } + extraPath := path.Join(testDir2, test.extraDir) + err = os.MkdirAll(extraPath, 0644) + if err != nil { + t.Fatalf("Error creating filesytem structure: %s", err) + } + + hash1, err := hashDirectory(testDir1) + if err != nil { + t.Fatalf("Failed to calculate hash: %s", err) + } + hash2, err := hashDirectory(testDir2) + if err != nil { + t.Fatalf("Failed to calculate hash: %s", err) + } + + if hash1 == hash2 { + t.Errorf("Expected different hashes, got: %s and %s", hash1, hash2) + } + }) + } +} + +func Test_CompositeKey_AddPath_WithExtraFilIgnored_Works(t *testing.T) { + tests := []struct { + name string + directories []string + files []string + extraFile string + }{ + { + name: "empty", + directories: []string{}, + files: []string{}, + extraFile: "extra", + }, + { + name: "dirs", + directories: []string{"foo", "bar", "foobar", "f/o/o"}, + files: []string{}, + extraFile: "f/o/o/extra", + }, + { + name: "files", + directories: []string{}, + files: []string{"foo", "bar", "foobar"}, + extraFile: "extra", + }, + { + name: "all", + directories: []string{"foo", "bar"}, + files: []string{"foo/bar", "bar/baz", "foobar"}, + extraFile: "bar/extra", + }, + } + + err := setIgnoreContext("**/extra") + if err != nil { + t.Fatalf("Error setting exlusion context: %s", err) + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testDir1, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Error creating tempdir: %s", err) + } + defer os.RemoveAll(testDir1) + err = createFilesystemStructure(testDir1, test.directories, test.files) + if err != nil { + t.Fatalf("Error creating filesytem structure: %s", err) + } + + testDir2, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Error creating tempdir: %s", err) + } + defer os.RemoveAll(testDir2) + err = createFilesystemStructure(testDir2, test.directories, test.files) + if err != nil { + t.Fatalf("Error creating filesytem structure: %s", err) + } + extraPath := path.Join(testDir2, test.extraFile) + err = ioutil.WriteFile(extraPath, []byte(test.extraFile), 0644) + if err != nil { + t.Fatalf("Error creating filesytem structure: %s", err) + } + + hash1, err := hashDirectory(testDir1) + if err != nil { + t.Fatalf("Failed to calculate hash: %s", err) + } + hash2, err := hashDirectory(testDir2) + if err != nil { + t.Fatalf("Failed to calculate hash: %s", err) + } + + if hash1 != hash2 { + t.Errorf("Expected equal hashes, got: %s and %s", hash1, hash2) + } + }) + } +} + +func Test_CompositeKey_AddPath_WithExtraDirIgnored_Works(t *testing.T) { + tests := []struct { + name string + directories []string + files []string + extraDir string + }{ + { + name: "empty", + directories: []string{}, + files: []string{}, + extraDir: "extra", + }, + { + name: "dirs", + directories: []string{"foo", "bar", "foobar", "f/o/o"}, + files: []string{}, + extraDir: "f/o/o/extra", + }, + { + name: "files", + directories: []string{}, + files: []string{"foo", "bar", "foobar"}, + extraDir: "extra", + }, + { + name: "all", + directories: []string{"foo", "bar"}, + files: []string{"foo/bar", "bar/baz", "foobar"}, + extraDir: "bar/extra", + }, + } + + err := setIgnoreContext("**/extra") + if err != nil { + t.Fatalf("Error setting exlusion context: %s", err) + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testDir1, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Error creating tempdir: %s", err) + } + defer os.RemoveAll(testDir1) + err = createFilesystemStructure(testDir1, test.directories, test.files) + if err != nil { + t.Fatalf("Error creating filesytem structure: %s", err) + } + + testDir2, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Error creating tempdir: %s", err) + } + defer os.RemoveAll(testDir2) + err = createFilesystemStructure(testDir2, test.directories, test.files) + if err != nil { + t.Fatalf("Error creating filesytem structure: %s", err) + } + extraPath := path.Join(testDir2, test.extraDir) + err = os.MkdirAll(extraPath, 0644) + if err != nil { + t.Fatalf("Error creating filesytem structure: %s", err) + } + + hash1, err := hashDirectory(testDir1) + if err != nil { + t.Fatalf("Failed to calculate hash: %s", err) + } + hash2, err := hashDirectory(testDir2) + if err != nil { + t.Fatalf("Failed to calculate hash: %s", err) + } + + if hash1 != hash2 { + t.Errorf("Expected equal hashes, got: %s and %s", hash1, hash2) + } + }) + } +} diff --git a/pkg/util/command_util.go b/pkg/util/command_util.go index 563d1a08d8..6f391e1335 100644 --- a/pkg/util/command_util.go +++ b/pkg/util/command_util.go @@ -213,7 +213,7 @@ func IsSrcsValid(srcsAndDest instructions.SourcesAndDest, resolvedSources []stri if !ContainsWildcards(srcs) { totalSrcs := 0 for _, src := range srcs { - if excludeFile(src, root) { + if ExcludeFile(src, root) { continue } totalSrcs++ @@ -250,7 +250,7 @@ func IsSrcsValid(srcsAndDest instructions.SourcesAndDest, resolvedSources []stri return errors.Wrap(err, "failed to get relative files") } for _, file := range files { - if excludeFile(file, root) { + if ExcludeFile(file, root) { continue } totalFiles++ diff --git a/pkg/util/fs_util.go b/pkg/util/fs_util.go index 437b045680..48d967da87 100644 --- a/pkg/util/fs_util.go +++ b/pkg/util/fs_util.go @@ -547,7 +547,7 @@ func CopyDir(src, dest, buildcontext string) ([]string, error) { fmt.Println(" i am returning from here this", err) return nil, err } - if excludeFile(fullPath, buildcontext) { + if ExcludeFile(fullPath, buildcontext) { logrus.Debugf("%s found in .dockerignore, ignoring", src) continue } @@ -580,7 +580,7 @@ func CopyDir(src, dest, buildcontext string) ([]string, error) { // CopySymlink copies the symlink at src to dest func CopySymlink(src, dest, buildcontext string) (bool, error) { - if excludeFile(src, buildcontext) { + if ExcludeFile(src, buildcontext) { logrus.Debugf("%s found in .dockerignore, ignoring", src) return true, nil } @@ -601,7 +601,7 @@ func CopySymlink(src, dest, buildcontext string) (bool, error) { // CopyFile copies the file at src to dest func CopyFile(src, dest, buildcontext string) (bool, error) { - if excludeFile(src, buildcontext) { + if ExcludeFile(src, buildcontext) { logrus.Debugf("%s found in .dockerignore, ignoring", src) return true, nil } @@ -645,8 +645,8 @@ func GetExcludedFiles(dockerfilepath string, buildcontext string) error { return err } -// excludeFile returns true if the .dockerignore specified this file should be ignored -func excludeFile(path, buildcontext string) bool { +// ExcludeFile returns true if the .dockerignore specified this file should be ignored +func ExcludeFile(path, buildcontext string) bool { if HasFilepathPrefix(path, buildcontext, false) { var err error path, err = filepath.Rel(buildcontext, path) diff --git a/pkg/util/fs_util_test.go b/pkg/util/fs_util_test.go index 3a97f72187..62e8b2b321 100644 --- a/pkg/util/fs_util_test.go +++ b/pkg/util/fs_util_test.go @@ -921,14 +921,14 @@ func Test_correctDockerignoreFileIsUsed(t *testing.T) { } for _, excl := range tt.args.excluded { t.Run(tt.name+" to exclude "+excl, func(t *testing.T) { - if !excludeFile(excl, tt.args.buildcontext) { + if !ExcludeFile(excl, tt.args.buildcontext) { t.Errorf("'%v' not excluded", excl) } }) } for _, incl := range tt.args.included { t.Run(tt.name+" to include "+incl, func(t *testing.T) { - if excludeFile(incl, tt.args.buildcontext) { + if ExcludeFile(incl, tt.args.buildcontext) { t.Errorf("'%v' not included", incl) } })