Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Draft]hack version where run now uses find. no whiteouts support #1274

Closed
wants to merge 1 commit into from
Closed

[Draft]hack version where run now uses find. no whiteouts support #1274

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 27 additions & 2 deletions pkg/commands/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,10 @@ import (
"os"
"os/exec"
"os/user"
"path/filepath"
"strings"
"syscall"
"time"

kConfig "github.com/GoogleContainerTools/kaniko/pkg/config"
"github.com/GoogleContainerTools/kaniko/pkg/constants"
Expand All @@ -36,7 +38,8 @@ import (

type RunCommand struct {
BaseCommand
cmd *instructions.RunCommand
cmd *instructions.RunCommand
Files []string
}

// for testing
Expand Down Expand Up @@ -64,6 +67,13 @@ func (r *RunCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bui
logrus.Infof("cmd: %s", newCommand[0])
logrus.Infof("args: %s", newCommand[1:])

// run command `touch filemarker`
file, _ := os.Create("/kaniko/marker.txt")
file.Close()
defer func() {
os.Remove("/kaniko/marker.txt")
}()

cmd := exec.Command(newCommand[0], newCommand[1:]...)

cmd.Dir = setWorkDirIfExists(config.WorkingDir)
Expand Down Expand Up @@ -112,6 +122,21 @@ func (r *RunCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bui
return err
}

// run command find to find all new files generated
t1 := time.Now().Second()
find := exec.Command("find", "/", "-newer", "/kaniko/marker.txt")
out, err := find.Output()
r.Files = []string{}
s := strings.Split(string(out), "\n")
for _, path := range s {
path = filepath.Clean(path)
if util.IsDestDir(path) || util.CheckWhitelist(path) {
continue
}
r.Files = append(r.Files, path)
}
t2 := time.Now().Second()
logrus.Infof("%d files changed found in %d seconds", len(r.Files), t2-t1)
return nil
}

Expand Down Expand Up @@ -149,7 +174,7 @@ func (r *RunCommand) String() string {
}

func (r *RunCommand) FilesToSnapshot() []string {
return nil
return r.Files
}

func (r *RunCommand) ProvidesFilesToSnapshot() bool {
Expand Down
63 changes: 31 additions & 32 deletions pkg/snapshot/snapshot.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@ import (
"os"
"path/filepath"
"sort"
"strings"
"syscall"
"time"

"github.com/GoogleContainerTools/kaniko/pkg/filesystem"
"github.com/GoogleContainerTools/kaniko/pkg/timing"
Expand Down Expand Up @@ -74,8 +76,16 @@ func (s *Snapshotter) TakeSnapshot(files []string) (string, error) {
logrus.Info("No files changed in this command, skipping snapshotting.")
return "", nil
}

filesToAdd, err := filesystem.ResolvePaths(files, s.whitelist)
whiteouts := []string{}
filesAdded := []string{}
for _, file := range files {
if strings.HasPrefix(file, "!") {
whiteouts = append(whiteouts, file)
} else {
filesAdded = append(filesAdded, file)
}
}
filesToAdd, err := filesystem.ResolvePaths(filesAdded, s.whitelist)
if err != nil {
return "", nil
}
Expand All @@ -94,7 +104,7 @@ func (s *Snapshotter) TakeSnapshot(files []string) (string, error) {

t := util.NewTar(f)
defer t.Close()
if err := writeToTar(t, filesToAdd, nil); err != nil {
if err := writeToTar(t, filesToAdd, whiteouts); err != nil {
return "", err
}
return f.Name(), nil
Expand Down Expand Up @@ -125,6 +135,7 @@ func (s *Snapshotter) TakeSnapshotFS() (string, error) {
func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) {
logrus.Info("Taking snapshot of full filesystem...")

timer1 := time.Now().Second()
// Some of the operations that follow (e.g. hashing) depend on the file system being synced,
// for example the hashing function that determines if files are equal uses the mtime of the files,
// which can lag if sync is not called. Unfortunately there can still be lag if too much data needs
Expand Down Expand Up @@ -158,40 +169,13 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) {
)
timing.DefaultRun.Stop(timer)

resolvedFiles, err := filesystem.ResolvePaths(foundPaths, s.whitelist)
if err != nil {
return nil, nil, err
}

resolvedMemFs := make(map[string]bool)
for _, f := range resolvedFiles {
resolvedMemFs[f] = true
}

// First handle whiteouts
// Get a list of all the files that existed before this layer
existingPaths := s.l.getFlattenedPathsForWhiteOut()

// Find the delta by removing everything left in this layer.
for p := range resolvedMemFs {
delete(existingPaths, p)
}

// The paths left here are the ones that have been deleted in this layer.
filesToWhiteOut := []string{}
for path := range existingPaths {
// Only add the whiteout if the directory for the file still exists.
dir := filepath.Dir(path)
if _, ok := resolvedMemFs[dir]; ok {
if s.l.MaybeAddWhiteout(path) {
logrus.Debugf("Adding whiteout for %s", path)
filesToWhiteOut = append(filesToWhiteOut, path)
}
}
}

filesToAdd := []string{}
for path := range resolvedMemFs {
for _, path := range foundPaths {
delete(existingPaths, path)
if util.CheckWhitelist(path) {
logrus.Tracef("Not adding %s to layer, as it's whitelisted", path)
continue
Expand All @@ -206,8 +190,23 @@ func (s *Snapshotter) scanFullFilesystem() ([]string, []string, error) {
filesToAdd = append(filesToAdd, path)
}
}
timer3 := time.Now().Second()
logrus.Infof("Resolve, %d files found in %d ", len(filesToAdd), timer3-timer1)

sort.Strings(filesToAdd)

// The paths left here are the ones that have been deleted in this layer.
filesToWhiteOut := []string{}
for path := range existingPaths {
// Only add the whiteout if the directory for the file still exists.
dir := filepath.Dir(path)
if _, ok := existingPaths[dir]; !ok {
if s.l.MaybeAddWhiteout(path) {
logrus.Debugf("Adding whiteout for %s", path)
filesToWhiteOut = append(filesToWhiteOut, path)
}
}
}
// Add files to the layered map
for _, file := range filesToAdd {
if err := s.l.Add(file); err != nil {
Expand Down