From 41ff94bb1bc72145ddee3cd4203359779a0df821 Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Thu, 17 Mar 2016 10:22:15 +0000 Subject: [PATCH 01/41] Renamed c -> command --- src/build/build.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/build/build.go b/src/build/build.go index 4d3d5cee..3200a347 100644 --- a/src/build/build.go +++ b/src/build/build.go @@ -94,12 +94,12 @@ func New(client Client, rockerfile *Rockerfile, cache Cache, cfg Config) *Build func (b *Build) Run(plan Plan) (err error) { for k := 0; k < len(plan); k++ { - c := plan[k] + command := plan[k] - log.Debugf("Step %d: %# v", k+1, pretty.Formatter(c)) + log.Debugf("Step %d: %# v", k+1, pretty.Formatter(command)) var doRun bool - if doRun, err = c.ShouldRun(b); err != nil { + if doRun, err = command.ShouldRun(b); err != nil { return err } if !doRun { @@ -107,13 +107,13 @@ func (b *Build) Run(plan Plan) (err error) { } // Replace env for the command if appropriate - if c, ok := c.(EnvReplacableCommand); ok { - c.ReplaceEnv(b.state.Config.Env) + if command, ok := command.(EnvReplacableCommand); ok { + command.ReplaceEnv(b.state.Config.Env) } - log.Infof("%s", color.New(color.FgWhite, color.Bold).SprintFunc()(c)) + log.Infof("%s", color.New(color.FgWhite, color.Bold).SprintFunc()(command)) - if b.state, err = c.Execute(b); err != nil { + if b.state, err = command.Execute(b); err != nil { return err } From c2f4e39a84e3f149f834be2b3477eb04bacb8fd5 Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Fri, 18 Mar 2016 13:22:24 +0000 Subject: [PATCH 02/41] Tests will clean after itselves --- test/mount_test.go | 2 +- test/utils.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/mount_test.go b/test/mount_test.go index 85d69c48..7ca4f0b5 100644 --- a/test/mount_test.go +++ b/test/mount_test.go @@ -12,7 +12,7 @@ func TestMountLocal(t *testing.T) { if err != nil { t.Fatalf("Can't create temp dir, err : %v", err) } - defer os.Remove(dir) + defer os.RemoveAll(dir) err = runRockerBuildWithOptions("FROM alpine:latest\n"+ "MOUNT "+dir+":/datadir\n"+ diff --git a/test/utils.go b/test/utils.go index ed22a2fa..49da27fb 100644 --- a/test/utils.go +++ b/test/utils.go @@ -94,7 +94,7 @@ func runRockerBuildWithOptions(content string, opts ...string) error { if err != nil { return err } - //defer os.Remove(filename) + defer os.RemoveAll(filename) gopath := getGOPATH() From 0a5b1ccc08f963e8d1014c77d723c9f8f662bc51 Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Fri, 18 Mar 2016 13:59:04 +0000 Subject: [PATCH 03/41] Added simple export test --- test/export_test.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 test/export_test.go diff --git a/test/export_test.go b/test/export_test.go new file mode 100644 index 00000000..d3e69bb3 --- /dev/null +++ b/test/export_test.go @@ -0,0 +1,29 @@ +package tests + +import ( + "github.com/stretchr/testify/assert" + "io/ioutil" + "os" + "testing" +) + +func TestExportSimple(t *testing.T) { + dir, err := ioutil.TempDir("/tmp", "rocker_integration_test_export") + assert.Nil(t, err, "Can't create tmp dir") + os.RemoveAll(dir) + + err = runRockerBuildWithOptions(` + FROM alpine:latest + RUN echo -n "test_export" > /exported_file + EXPORT /exported_file + + FROM alpine:latest + MOUNT `+dir+`:/datadir + IMPORT /exported_file /datadir/imported_file`, "--no-cache") + assert.Nil(t, err, "Failed to run rocker build") + + content, err := ioutil.ReadFile(dir + "/imported_file") + assert.Nil(t, err, "Can't read file") + + assert.Equal(t, "test_export", string(content)) +} From f42e1bd8244a22dbf2ba80a8b369bbfc68dd6e3d Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Fri, 18 Mar 2016 14:35:07 +0000 Subject: [PATCH 04/41] Implement test for Smolin's issue --- test/export_test.go | 58 +++++++++++++++++++++++++++++++++++++++++++-- test/utils.go | 12 +++++++++- 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/test/export_test.go b/test/export_test.go index d3e69bb3..7ff4d0f6 100644 --- a/test/export_test.go +++ b/test/export_test.go @@ -8,9 +8,9 @@ import ( ) func TestExportSimple(t *testing.T) { - dir, err := ioutil.TempDir("/tmp", "rocker_integration_test_export") + dir, err := ioutil.TempDir("/tmp", "rocker_integration_test_export_") assert.Nil(t, err, "Can't create tmp dir") - os.RemoveAll(dir) + defer os.RemoveAll(dir) err = runRockerBuildWithOptions(` FROM alpine:latest @@ -27,3 +27,57 @@ func TestExportSimple(t *testing.T) { assert.Equal(t, "test_export", string(content)) } + +func TestExportSmolinIssue(t *testing.T) { + dir, err := ioutil.TempDir("/tmp", "rocker_integration_test_export_") + assert.Nil(t, err, "Can't create tmp dir") + defer os.RemoveAll(dir) + + rockerfile, err := createTempFile("") + assert.Nil(t, err, "Can't create temp file") + defer os.RemoveAll(rockerfile) + + rocker_content_first := []byte(`FROM alpine + RUN echo -n "first" > /exported_file + EXPORT /exported_file + FROM alpine + MOUNT ` + dir + `:/datadir + IMPORT /exported_file /datadir/imported_file`) + + rocker_content_second := []byte(`FROM alpine + RUN echo -n "second" > /exported_file + EXPORT /exported_file + FROM alpine + MOUNT ` + dir + `:/datadir + IMPORT /exported_file /datadir/imported_file`) + + err = ioutil.WriteFile(rockerfile, rocker_content_first, 0644) + assert.Nil(t, err) + + err = runRockerBuildWithFile(rockerfile) + assert.Nil(t, err) + + content, err := ioutil.ReadFile(dir + "/imported_file") + assert.Nil(t, err) + assert.Equal(t, "first", string(content)) + + err = ioutil.WriteFile(rockerfile, rocker_content_second, 0644) + assert.Nil(t, err) + + err = runRockerBuildWithFile(rockerfile) + assert.Nil(t, err) + + content, err = ioutil.ReadFile(dir + "/imported_file") + assert.Nil(t, err) + assert.Equal(t, "second", string(content)) + + err = ioutil.WriteFile(rockerfile, rocker_content_first, 0644) + assert.Nil(t, err) + + err = runRockerBuildWithFile(rockerfile) + assert.Nil(t, err) + + content, err = ioutil.ReadFile(dir + "/imported_file") + assert.Nil(t, err, "Can't read file") + assert.Equal(t, "first", string(content)) +} diff --git a/test/utils.go b/test/utils.go index 49da27fb..9be728b4 100644 --- a/test/utils.go +++ b/test/utils.go @@ -89,6 +89,17 @@ func createTempFile(content string) (string, error) { return tmpfile.Name(), nil } +func runRockerBuildWithFile(filename string, opts ...string) error { + gopath := getGOPATH() + + p := []string{"build", "-f", filename} + params := append(p, opts...) + if err := runCmd(gopath+"/bin/rocker", nil, params...); err != nil { + return err + } + + return nil +} func runRockerBuildWithOptions(content string, opts ...string) error { filename, err := createTempFile(content) if err != nil { @@ -101,7 +112,6 @@ func runRockerBuildWithOptions(content string, opts ...string) error { p := []string{"build", "-f", filename} params := append(p, opts...) if err := runCmd(gopath+"/bin/rocker", nil, params...); err != nil { - //fmt.Printf("Failed to run rocker with filename '%v'", filename) return err } From bfd4deeb148b5e5beec1c92cc5d6739c70a7c6b1 Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Fri, 18 Mar 2016 14:44:25 +0000 Subject: [PATCH 05/41] Make lint happy --- test/export_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/export_test.go b/test/export_test.go index 7ff4d0f6..41a94064 100644 --- a/test/export_test.go +++ b/test/export_test.go @@ -37,21 +37,21 @@ func TestExportSmolinIssue(t *testing.T) { assert.Nil(t, err, "Can't create temp file") defer os.RemoveAll(rockerfile) - rocker_content_first := []byte(`FROM alpine + rockerContentFirst := []byte(`FROM alpine RUN echo -n "first" > /exported_file EXPORT /exported_file FROM alpine MOUNT ` + dir + `:/datadir IMPORT /exported_file /datadir/imported_file`) - rocker_content_second := []byte(`FROM alpine + rockerContentSecond := []byte(`FROM alpine RUN echo -n "second" > /exported_file EXPORT /exported_file FROM alpine MOUNT ` + dir + `:/datadir IMPORT /exported_file /datadir/imported_file`) - err = ioutil.WriteFile(rockerfile, rocker_content_first, 0644) + err = ioutil.WriteFile(rockerfile, rockerContentFirst, 0644) assert.Nil(t, err) err = runRockerBuildWithFile(rockerfile) @@ -61,7 +61,7 @@ func TestExportSmolinIssue(t *testing.T) { assert.Nil(t, err) assert.Equal(t, "first", string(content)) - err = ioutil.WriteFile(rockerfile, rocker_content_second, 0644) + err = ioutil.WriteFile(rockerfile, rockerContentSecond, 0644) assert.Nil(t, err) err = runRockerBuildWithFile(rockerfile) @@ -71,7 +71,7 @@ func TestExportSmolinIssue(t *testing.T) { assert.Nil(t, err) assert.Equal(t, "second", string(content)) - err = ioutil.WriteFile(rockerfile, rocker_content_first, 0644) + err = ioutil.WriteFile(rockerfile, rockerContentFirst, 0644) assert.Nil(t, err) err = runRockerBuildWithFile(rockerfile) From 212fcab093a22a90264d72ca02b4369740d0bcbb Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Fri, 18 Mar 2016 14:46:55 +0000 Subject: [PATCH 06/41] Will skip Smolin's test for now. Will enable it back when the issue is fixed in rocker --- test/export_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/export_test.go b/test/export_test.go index 41a94064..938c7898 100644 --- a/test/export_test.go +++ b/test/export_test.go @@ -29,6 +29,7 @@ func TestExportSimple(t *testing.T) { } func TestExportSmolinIssue(t *testing.T) { + t.Skip() dir, err := ioutil.TempDir("/tmp", "rocker_integration_test_export_") assert.Nil(t, err, "Can't create tmp dir") defer os.RemoveAll(dir) From 4dadd7e151f715280f1862ede39e64d121c691ba Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Fri, 18 Mar 2016 14:58:55 +0000 Subject: [PATCH 07/41] Will reload cache at first run of rocker in Smolin's test --- test/export_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/export_test.go b/test/export_test.go index 938c7898..ad9482dc 100644 --- a/test/export_test.go +++ b/test/export_test.go @@ -28,6 +28,9 @@ func TestExportSimple(t *testing.T) { assert.Equal(t, "test_export", string(content)) } +func TestExportMultipleFrom(t *testing.T) { +} + func TestExportSmolinIssue(t *testing.T) { t.Skip() dir, err := ioutil.TempDir("/tmp", "rocker_integration_test_export_") @@ -55,7 +58,7 @@ func TestExportSmolinIssue(t *testing.T) { err = ioutil.WriteFile(rockerfile, rockerContentFirst, 0644) assert.Nil(t, err) - err = runRockerBuildWithFile(rockerfile) + err = runRockerBuildWithFile(rockerfile, "--reload-cache") assert.Nil(t, err) content, err := ioutil.ReadFile(dir + "/imported_file") From f5e2ea528d0972d16cdbb56feb2aea29e783bb17 Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Fri, 18 Mar 2016 16:15:58 +0000 Subject: [PATCH 08/41] Added TestExportSeparateFiles test. --- test/export_test.go | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/test/export_test.go b/test/export_test.go index ad9482dc..70a8a2a9 100644 --- a/test/export_test.go +++ b/test/export_test.go @@ -28,7 +28,37 @@ func TestExportSimple(t *testing.T) { assert.Equal(t, "test_export", string(content)) } -func TestExportMultipleFrom(t *testing.T) { +func TestExportSeparateFiles(t *testing.T) { + dir, err := ioutil.TempDir("/tmp", "rocker_integration_test_export_") + assert.Nil(t, err, "Can't create tmp dir") + defer os.RemoveAll(dir) + + rockerContentFirst := `FROM alpine:latest + EXPORT /etc/issue + RUN echo -n "first_separate" > /exported_file + EXPORT /exported_file + + FROM alpine + MOUNT ` + dir + `:/datadir + IMPORT /exported_file /datadir/imported_file` + + rockerContentSecond := `FROM alpine:latest + EXPORT /etc/issue + RUN echo -n "second_separate" > /exported_file + EXPORT /exported_file ` + + err = runRockerBuildWithOptions(rockerContentFirst, "--reload-cache") + assert.Nil(t, err) + + err = runRockerBuildWithOptions(rockerContentSecond) + assert.Nil(t, err) + + err = runRockerBuildWithOptions(rockerContentFirst) + assert.Nil(t, err) + + content, err := ioutil.ReadFile(dir + "/imported_file") + assert.Nil(t, err) + assert.Equal(t, "first_separate", string(content)) } func TestExportSmolinIssue(t *testing.T) { From f9d10c5d9f430d3effd6196e39480b1db70e8c0f Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Fri, 18 Mar 2016 16:33:30 +0000 Subject: [PATCH 09/41] Added test for few FROM's --- test/export_test.go | 46 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/test/export_test.go b/test/export_test.go index 70a8a2a9..cd22a7a1 100644 --- a/test/export_test.go +++ b/test/export_test.go @@ -115,3 +115,49 @@ func TestExportSmolinIssue(t *testing.T) { assert.Nil(t, err, "Can't read file") assert.Equal(t, "first", string(content)) } + +func TestExportSameFileFewFroms(t *testing.T) { + dir, err := ioutil.TempDir("/tmp", "rocker_integration_test_export_") + assert.Nil(t, err, "Can't create tmp dir") + defer os.RemoveAll(dir) + + rockerfile, err := createTempFile("") + assert.Nil(t, err, "Can't create temp file") + defer os.RemoveAll(rockerfile) + + rockerContentFirst := []byte(`FROM alpine + EXPORT /etc/issue + + FROM alpine + RUN echo -n "first_few" > /exported_file + EXPORT /exported_file + + FROM alpine + MOUNT ` + dir + `:/datadir + IMPORT /exported_file /datadir/imported_file`) + + rockerContentSecond := []byte(`FROM alpine + EXPORT /etc/issue + + FROM alpine + RUN echo -n "second_few" > /exported_file + EXPORT /exported_file`) + + err = ioutil.WriteFile(rockerfile, rockerContentFirst, 0644) + assert.Nil(t, err) + err = runRockerBuildWithFile(rockerfile, "--reload-cache") + assert.Nil(t, err) + + err = ioutil.WriteFile(rockerfile, rockerContentSecond, 0644) + assert.Nil(t, err) + err = runRockerBuildWithFile(rockerfile, "--reload-cache") + assert.Nil(t, err) + + err = ioutil.WriteFile(rockerfile, rockerContentFirst, 0644) + assert.Nil(t, err) + err = runRockerBuildWithFile(rockerfile) + assert.Nil(t, err) + content, err := ioutil.ReadFile(dir + "/imported_file") + assert.Nil(t, err, "Can't read file") + assert.Equal(t, "first_few", string(content)) +} From 92c5dfdbf14cacc4c45930ef57f872b86bfb1c7a Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Fri, 18 Mar 2016 16:34:13 +0000 Subject: [PATCH 10/41] Test for few FROM's will be skipped for now. --- test/export_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/export_test.go b/test/export_test.go index cd22a7a1..bc211880 100644 --- a/test/export_test.go +++ b/test/export_test.go @@ -117,6 +117,7 @@ func TestExportSmolinIssue(t *testing.T) { } func TestExportSameFileFewFroms(t *testing.T) { + t.Skip() dir, err := ioutil.TempDir("/tmp", "rocker_integration_test_export_") assert.Nil(t, err, "Can't create tmp dir") defer os.RemoveAll(dir) From d614b433af9616a588db3c4eac3c5bf52f9544f7 Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Fri, 18 Mar 2016 16:38:13 +0000 Subject: [PATCH 11/41] Better test formatting --- test/export_test.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/export_test.go b/test/export_test.go index bc211880..ec3a92f0 100644 --- a/test/export_test.go +++ b/test/export_test.go @@ -87,30 +87,24 @@ func TestExportSmolinIssue(t *testing.T) { err = ioutil.WriteFile(rockerfile, rockerContentFirst, 0644) assert.Nil(t, err) - err = runRockerBuildWithFile(rockerfile, "--reload-cache") assert.Nil(t, err) - content, err := ioutil.ReadFile(dir + "/imported_file") assert.Nil(t, err) assert.Equal(t, "first", string(content)) err = ioutil.WriteFile(rockerfile, rockerContentSecond, 0644) assert.Nil(t, err) - err = runRockerBuildWithFile(rockerfile) assert.Nil(t, err) - content, err = ioutil.ReadFile(dir + "/imported_file") assert.Nil(t, err) assert.Equal(t, "second", string(content)) err = ioutil.WriteFile(rockerfile, rockerContentFirst, 0644) assert.Nil(t, err) - err = runRockerBuildWithFile(rockerfile) assert.Nil(t, err) - content, err = ioutil.ReadFile(dir + "/imported_file") assert.Nil(t, err, "Can't read file") assert.Equal(t, "first", string(content)) From fd22d6109a96cfe99c790d65792dbc1f0207e648 Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Fri, 18 Mar 2016 17:18:40 +0000 Subject: [PATCH 12/41] Added TestExportSeparateFilesDifferentExport --- test/export_test.go | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/test/export_test.go b/test/export_test.go index ec3a92f0..ec2b6d1f 100644 --- a/test/export_test.go +++ b/test/export_test.go @@ -28,7 +28,40 @@ func TestExportSimple(t *testing.T) { assert.Equal(t, "test_export", string(content)) } -func TestExportSeparateFiles(t *testing.T) { +func TestExportSeparateFilesDifferentExport(t *testing.T) { + dir, err := ioutil.TempDir("/tmp", "rocker_integration_test_export_") + assert.Nil(t, err, "Can't create tmp dir") + defer os.RemoveAll(dir) + + rockerContentFirst := `FROM alpine:latest + EXPORT /etc/hostname + RUN echo -n "first_diff" > /exported_file + EXPORT /exported_file + + FROM alpine + MOUNT ` + dir + `:/datadir + IMPORT /exported_file /datadir/imported_file` + + rockerContentSecond := `FROM alpine:latest + EXPORT /etc/issue + RUN echo -n "second_diff" > /exported_file + EXPORT /exported_file ` + + err = runRockerBuildWithOptions(rockerContentFirst, "--reload-cache") + assert.Nil(t, err) + + err = runRockerBuildWithOptions(rockerContentSecond, "--reload-cache") + assert.Nil(t, err) + + err = runRockerBuildWithOptions(rockerContentFirst) + assert.Nil(t, err) + + content, err := ioutil.ReadFile(dir + "/imported_file") + assert.Nil(t, err) + assert.Equal(t, "first_diff", string(content)) +} + +func TestExportSeparateFilesSameExport(t *testing.T) { dir, err := ioutil.TempDir("/tmp", "rocker_integration_test_export_") assert.Nil(t, err, "Can't create tmp dir") defer os.RemoveAll(dir) @@ -50,7 +83,7 @@ func TestExportSeparateFiles(t *testing.T) { err = runRockerBuildWithOptions(rockerContentFirst, "--reload-cache") assert.Nil(t, err) - err = runRockerBuildWithOptions(rockerContentSecond) + err = runRockerBuildWithOptions(rockerContentSecond, "--reload-cache") assert.Nil(t, err) err = runRockerBuildWithOptions(rockerContentFirst) From 1250e899dcb73d6f35a8157f19372ea311713f9e Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Tue, 22 Mar 2016 13:08:32 +0000 Subject: [PATCH 13/41] Fixed tests --- test/export_test.go | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/test/export_test.go b/test/export_test.go index ec2b6d1f..0afc6b4a 100644 --- a/test/export_test.go +++ b/test/export_test.go @@ -45,7 +45,11 @@ func TestExportSeparateFilesDifferentExport(t *testing.T) { rockerContentSecond := `FROM alpine:latest EXPORT /etc/issue RUN echo -n "second_diff" > /exported_file - EXPORT /exported_file ` + EXPORT /exported_file + + FROM alpine + MOUNT ` + dir + `:/datadir + IMPORT /exported_file /datadir/imported_file` err = runRockerBuildWithOptions(rockerContentFirst, "--reload-cache") assert.Nil(t, err) @@ -78,12 +82,16 @@ func TestExportSeparateFilesSameExport(t *testing.T) { rockerContentSecond := `FROM alpine:latest EXPORT /etc/issue RUN echo -n "second_separate" > /exported_file - EXPORT /exported_file ` + EXPORT /exported_file + + FROM alpine + MOUNT ` + dir + `:/datadir + IMPORT /exported_file /datadir/imported_file` err = runRockerBuildWithOptions(rockerContentFirst, "--reload-cache") assert.Nil(t, err) - err = runRockerBuildWithOptions(rockerContentSecond, "--reload-cache") + err = runRockerBuildWithOptions(rockerContentSecond) assert.Nil(t, err) err = runRockerBuildWithOptions(rockerContentFirst) @@ -95,7 +103,6 @@ func TestExportSeparateFilesSameExport(t *testing.T) { } func TestExportSmolinIssue(t *testing.T) { - t.Skip() dir, err := ioutil.TempDir("/tmp", "rocker_integration_test_export_") assert.Nil(t, err, "Can't create tmp dir") defer os.RemoveAll(dir) @@ -105,26 +112,34 @@ func TestExportSmolinIssue(t *testing.T) { defer os.RemoveAll(rockerfile) rockerContentFirst := []byte(`FROM alpine - RUN echo -n "first" > /exported_file + RUN echo -n "first_smolin" > /exported_file EXPORT /exported_file FROM alpine MOUNT ` + dir + `:/datadir IMPORT /exported_file /datadir/imported_file`) rockerContentSecond := []byte(`FROM alpine - RUN echo -n "second" > /exported_file + RUN echo -n "second_smolin" > /exported_file EXPORT /exported_file FROM alpine MOUNT ` + dir + `:/datadir IMPORT /exported_file /datadir/imported_file`) + rockerContentThird := []byte(`FROM alpine + RUN echo -n "first_smolin" > /exported_file + EXPORT /exported_file + FROM alpine + MOUNT ` + dir + `:/datadir + RUN echo "Invalidate cache" + IMPORT /exported_file /datadir/imported_file`) + err = ioutil.WriteFile(rockerfile, rockerContentFirst, 0644) assert.Nil(t, err) err = runRockerBuildWithFile(rockerfile, "--reload-cache") assert.Nil(t, err) content, err := ioutil.ReadFile(dir + "/imported_file") assert.Nil(t, err) - assert.Equal(t, "first", string(content)) + assert.Equal(t, "first_smolin", string(content)) err = ioutil.WriteFile(rockerfile, rockerContentSecond, 0644) assert.Nil(t, err) @@ -132,19 +147,18 @@ func TestExportSmolinIssue(t *testing.T) { assert.Nil(t, err) content, err = ioutil.ReadFile(dir + "/imported_file") assert.Nil(t, err) - assert.Equal(t, "second", string(content)) + assert.Equal(t, "second_smolin", string(content)) - err = ioutil.WriteFile(rockerfile, rockerContentFirst, 0644) + err = ioutil.WriteFile(rockerfile, rockerContentThird, 0644) assert.Nil(t, err) err = runRockerBuildWithFile(rockerfile) assert.Nil(t, err) content, err = ioutil.ReadFile(dir + "/imported_file") assert.Nil(t, err, "Can't read file") - assert.Equal(t, "first", string(content)) + assert.Equal(t, "first_smolin", string(content)) } func TestExportSameFileFewFroms(t *testing.T) { - t.Skip() dir, err := ioutil.TempDir("/tmp", "rocker_integration_test_export_") assert.Nil(t, err, "Can't create tmp dir") defer os.RemoveAll(dir) From bb9eb659d251c97cd95234b8a84f6e8ba74b5e36 Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Tue, 22 Mar 2016 14:52:43 +0000 Subject: [PATCH 14/41] exportsContainerName will return name based on commit and imageSha --- src/build/util.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/build/util.go b/src/build/util.go index d336feb2..2a9f6555 100644 --- a/src/build/util.go +++ b/src/build/util.go @@ -31,12 +31,6 @@ func (b *Build) mountsContainerName(path string) string { return fmt.Sprintf("rocker_mount_%.6x", md5.Sum([]byte(mountID))) } -// exportsContainerName return the name of volume container that will be used for EXPORTs -func (b *Build) exportsContainerName() string { - mountID := b.getIdentifier() - return fmt.Sprintf("rocker_exports_%.6x", md5.Sum([]byte(mountID))) -} - // getIdentifier returns the sequence that is unique to the current Rockerfile func (b *Build) getIdentifier() string { if b.cfg.ID != "" { @@ -55,6 +49,13 @@ func mountsToBinds(mounts []docker.Mount) []string { return result } +// exportsContainerName return the name of volume container that will be used for EXPORTs +func exportsContainerName(imageID string, commits string) string { + mountID := imageID + commits + name := fmt.Sprintf("rocker_exports_%.12x", md5.Sum([]byte(mountID))) + return name +} + // mountToBind turns docker.Mount into a bind string func mountToBind(m docker.Mount, rw bool) string { if rw { From ad7d77cc2b055b36dc2e57c51521f2f23c75bf18 Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Tue, 22 Mar 2016 14:54:07 +0000 Subject: [PATCH 15/41] Will use commit and imgSha as container name. Each next EXPORT will depend on previous --- src/build/build.go | 6 ++++-- src/build/commands.go | 20 +++++++++++++++----- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/build/build.go b/src/build/build.go index 3200a347..74897c40 100644 --- a/src/build/build.go +++ b/src/build/build.go @@ -75,6 +75,9 @@ type Build struct { // A little hack to support cross-FROM cache for EXPORTS // maybe rethink it later exports []string + + currentExportContainerName string + prevExportContainer string } // New creates the new build object @@ -228,8 +231,7 @@ func (b *Build) getVolumeContainer(path string) (c *docker.Container, err error) return b.client.InspectContainer(name) } -func (b *Build) getExportsContainer() (c *docker.Container, err error) { - name := b.exportsContainerName() +func (b *Build) getExportsContainer(name string) (c *docker.Container, err error) { config := &docker.Config{ Image: RsyncImage, diff --git a/src/build/commands.go b/src/build/commands.go index 29b40436..0ec19478 100644 --- a/src/build/commands.go +++ b/src/build/commands.go @@ -1192,7 +1192,14 @@ func (c *CommandExport) Execute(b *Build) (s State, err error) { // EXPORT /my/dir /stuff/ --> /EXPORT_VOLUME/stuff/my_dir // EXPORT /my/dir/* / --> /EXPORT_VOLUME/stuff/my_dir - exportsContainer, err := b.getExportsContainer() + s.Commit("EXPORT %q to %s[prev_export_container:%s]", src, dest, b.prevExportContainer) + + name := exportsContainerName(s.ImageID, s.GetCommits()) + if b.currentExportContainerName == "" { + b.currentExportContainerName = name + } + + exportsContainer, err := b.getExportsContainer(b.currentExportContainerName) if err != nil { return s, err } @@ -1203,14 +1210,13 @@ func (c *CommandExport) Execute(b *Build) (s State, err error) { return s, fmt.Errorf("Invalid EXPORT destination: %s", dest) } - s.Commit("EXPORT %q to %.12s:%s", src, exportsContainer.ID, dest) - s, hit, err := b.probeCache(s) if err != nil { return s, err } if hit { b.exports = append(b.exports, s.ExportsID) + b.prevExportContainer = s.ExportsID return s, nil } @@ -1222,6 +1228,7 @@ func (c *CommandExport) Execute(b *Build) (s State, err error) { s = origState s.ExportsID = exportsID b.exports = append(b.exports, exportsID) + b.prevExportContainer = exportsID }() // Append exports container as a volume @@ -1286,13 +1293,16 @@ func (c *CommandImport) Execute(b *Build) (s State, err error) { // because it was built earlier with the same prerequisites, but the actual // data in the exports container may be from the latest EXPORT of different // build. So we need to prefix ~/.rocker_exports dir with some id somehow. + if b.currentExportContainerName == "" { + return s, fmt.Errorf("You have to EXPORT something first to do IMPORT") + } - exportsContainer, err := b.getExportsContainer() + exportsContainer, err := b.getExportsContainer(b.currentExportContainerName) if err != nil { return s, err } - log.Infof("| Import from %s (%.12s)", b.exportsContainerName(), exportsContainer.ID) + log.Infof("| Import from %s (%.12s)", b.currentExportContainerName, exportsContainer.ID) // If only one argument was given to IMPORT, use the same path for destination // IMPORT /my/dir/file.tar --> ADD ./EXPORT_VOLUME/my/dir/file.tar /my/dir/file.tar From 57699dad2785f93628dc10d9255619371718ffb9 Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Tue, 22 Mar 2016 15:10:14 +0000 Subject: [PATCH 16/41] Each next EXPORT will be dependent on previous one Also, this allowed to make IMPORT dependent on all exports just be referring to last EXPORT. replaced b.exports with b.prevExportContainer --- src/build/commands.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/build/commands.go b/src/build/commands.go index 0ec19478..1cbb1429 100644 --- a/src/build/commands.go +++ b/src/build/commands.go @@ -1192,7 +1192,7 @@ func (c *CommandExport) Execute(b *Build) (s State, err error) { // EXPORT /my/dir /stuff/ --> /EXPORT_VOLUME/stuff/my_dir // EXPORT /my/dir/* / --> /EXPORT_VOLUME/stuff/my_dir - s.Commit("EXPORT %q to %s[prev_export_container:%s]", src, dest, b.prevExportContainer) + s.Commit("EXPORT %q to %s, prev_export_container: %s", src, dest, b.prevExportContainer) name := exportsContainerName(s.ImageID, s.GetCommits()) if b.currentExportContainerName == "" { @@ -1215,7 +1215,6 @@ func (c *CommandExport) Execute(b *Build) (s State, err error) { return s, err } if hit { - b.exports = append(b.exports, s.ExportsID) b.prevExportContainer = s.ExportsID return s, nil } @@ -1227,7 +1226,6 @@ func (c *CommandExport) Execute(b *Build) (s State, err error) { defer func() { s = origState s.ExportsID = exportsID - b.exports = append(b.exports, exportsID) b.prevExportContainer = exportsID }() @@ -1284,7 +1282,7 @@ func (c *CommandImport) Execute(b *Build) (s State, err error) { if len(args) == 0 { return s, fmt.Errorf("IMPORT requires at least one argument") } - if len(b.exports) == 0 { + if b.prevExportContainer == "" { return s, fmt.Errorf("You have to EXPORT something first in order to IMPORT") } @@ -1320,8 +1318,7 @@ func (c *CommandImport) Execute(b *Build) (s State, err error) { src = append(src, argResolved) } - sort.Strings(b.exports) - s.Commit("IMPORT %q : %q %s", b.exports, src, dest) + s.Commit("IMPORT %q : %q %s", b.prevExportContainer, src, dest) // Check cache s, hit, err := b.probeCache(s) From 1bd7ad5b730fec6f2796b857766dc7a2ea23660e Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Wed, 23 Mar 2016 12:56:39 +0000 Subject: [PATCH 17/41] Added prefix support to mountToBinds function --- src/build/util.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/build/util.go b/src/build/util.go index 2a9f6555..95a76cb9 100644 --- a/src/build/util.go +++ b/src/build/util.go @@ -40,11 +40,11 @@ func (b *Build) getIdentifier() string { } // mountsToBinds turns the list of mounts to the list of binds -func mountsToBinds(mounts []docker.Mount) []string { +func mountsToBinds(mounts []docker.Mount, prefix string) []string { result := make([]string, len(mounts)) for i, m := range mounts { // TODO: Mode? - result[i] = mountToBind(m, m.RW) + result[i] = mountToBind(m, m.RW, prefix) } return result } @@ -57,11 +57,11 @@ func exportsContainerName(imageID string, commits string) string { } // mountToBind turns docker.Mount into a bind string -func mountToBind(m docker.Mount, rw bool) string { +func mountToBind(m docker.Mount, rw bool, prefix string) string { if rw { - return m.Source + ":" + m.Destination + ":rw" + return m.Source + ":" + m.Destination + prefix + ":rw" } - return m.Source + ":" + m.Destination + ":ro" + return m.Source + ":" + m.Destination + prefix + ":ro" } // readerVoidCloser is a hack of the improved go-dockerclient's hijacking behavior From 053fd63922d2daa50a29983db997bb646bdc4bad Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Wed, 23 Mar 2016 12:57:25 +0000 Subject: [PATCH 18/41] Added IsCached flag --- src/build/state.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/build/state.go b/src/build/state.go index 3fcf1085..518d901a 100644 --- a/src/build/state.go +++ b/src/build/state.go @@ -46,6 +46,7 @@ type StateNoCache struct { CmdSet bool ContainerID string HostConfig docker.HostConfig + IsCached bool } // NewState makes a fresh state From a99991fb64ba5338af109968ee9235bc4bd3eddf Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Wed, 23 Mar 2016 12:58:49 +0000 Subject: [PATCH 19/41] Added ability to provide HostConfig as param to EnsureContainer --- src/build/client.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/build/client.go b/src/build/client.go index ce88a91a..5af8affc 100644 --- a/src/build/client.go +++ b/src/build/client.go @@ -56,7 +56,7 @@ type Client interface { CommitContainer(state State, message string) (img *docker.Image, err error) RemoveContainer(containerID string) error UploadToContainer(containerID string, stream io.Reader, path string) error - EnsureContainer(containerName string, config *docker.Config, purpose string) (containerID string, err error) + EnsureContainer(containerName string, config *docker.Config, hostConfig *docker.HostConfig, purpose string) (containerID string, err error) InspectContainer(containerName string) (*docker.Container, error) ResolveHostPath(path string) (resultPath string, err error) } @@ -576,7 +576,7 @@ func (c *DockerClient) EnsureImage(imageName string) (err error) { // EnsureContainer checks if container with specified name exists // and creates it otherwise -func (c *DockerClient) EnsureContainer(containerName string, config *docker.Config, purpose string) (containerID string, err error) { +func (c *DockerClient) EnsureContainer(containerName string, config *docker.Config, hostConfig *docker.HostConfig, purpose string) (containerID string, err error) { // Check if container exists container, err := c.client.InspectContainer(containerName) @@ -597,8 +597,9 @@ func (c *DockerClient) EnsureContainer(containerName string, config *docker.Conf c.log.Infof("| Create container: %s for %s", containerName, purpose) opts := docker.CreateContainerOptions{ - Name: containerName, - Config: config, + Name: containerName, + Config: config, + HostConfig: hostConfig, } c.log.Debugf("Create container options %# v", opts) From 391fb5c59df031c078883e3e5495e8a9630aa529 Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Wed, 23 Mar 2016 12:59:27 +0000 Subject: [PATCH 20/41] Implemented new EXPORT/IMPORT aproach --- src/build/build.go | 28 +++++++++++++---- src/build/commands.go | 72 ++++++++++++++++++++++++++++++------------- 2 files changed, 72 insertions(+), 28 deletions(-) diff --git a/src/build/build.go b/src/build/build.go index 74897c40..f9983e83 100644 --- a/src/build/build.go +++ b/src/build/build.go @@ -77,7 +77,7 @@ type Build struct { exports []string currentExportContainerName string - prevExportContainer string + prevExportContainerID string } // New creates the new build object @@ -156,6 +156,8 @@ func (b *Build) GetImageID() string { } func (b *Build) probeCache(s State) (cachedState State, hit bool, err error) { + s.NoCache.IsCached = false + if b.cache == nil || s.NoCache.CacheBusted { return s, false, nil } @@ -203,8 +205,9 @@ func (b *Build) probeCache(s State) (cachedState State, hit bool, err error) { // Keep items that should not be cached from the previous state s2.NoCache = s.NoCache - // We don't want commits to go through the cache - s2.CleanCommits() + + // Mark state as cached + s2.NoCache.IsCached = true return *s2, true, nil } @@ -222,7 +225,7 @@ func (b *Build) getVolumeContainer(path string) (c *docker.Container, err error) log.Debugf("Make MOUNT volume container %s with options %# v", name, config) - if _, err = b.client.EnsureContainer(name, config, path); err != nil { + if _, err = b.client.EnsureContainer(name, config, nil, path); err != nil { return nil, err } @@ -231,7 +234,7 @@ func (b *Build) getVolumeContainer(path string) (c *docker.Container, err error) return b.client.InspectContainer(name) } -func (b *Build) getExportsContainer(name string) (c *docker.Container, err error) { +func (b *Build) getExportsContainerWithBinds(name string, binds []string) (c *docker.Container, err error) { config := &docker.Config{ Image: RsyncImage, @@ -239,11 +242,21 @@ func (b *Build) getExportsContainer(name string) (c *docker.Container, err error "/opt/rsync/bin": struct{}{}, ExportsPath: struct{}{}, }, + Cmd: []string{"/opt/rsync/bin/rsync", "-a", "--delete-during", "/.rocker_exports_source/", "/.rocker_exports/"}, + Entrypoint: []string{}, + } + + var hostConfig *docker.HostConfig + + if len(binds) != 0 { + hostConfig = &docker.HostConfig{ + Binds: binds, + } } log.Debugf("Make EXPORT container %s with options %# v", name, config) - containerID, err := b.client.EnsureContainer(name, config, "exports") + containerID, err := b.client.EnsureContainer(name, config, hostConfig, "exports") if err != nil { return nil, err } @@ -252,6 +265,9 @@ func (b *Build) getExportsContainer(name string) (c *docker.Container, err error return b.client.InspectContainer(containerID) } +func (b *Build) getExportsContainer(name string) (c *docker.Container, err error) { + return b.getExportsContainerWithBinds(name, []string{}) +} // lookupImage looks up for the image by name and returns *docker.Image object (result of the inspect) // `Pull` config option defines whether we want to update the latest version of the image from the remote registry diff --git a/src/build/commands.go b/src/build/commands.go index 1cbb1429..33bf9867 100644 --- a/src/build/commands.go +++ b/src/build/commands.go @@ -277,17 +277,17 @@ func (c *CommandCommit) String() string { // ShouldRun returns true if the command should be executed func (c *CommandCommit) ShouldRun(b *Build) (bool, error) { - return b.state.GetCommits() != "", nil + return !b.state.NoCache.IsCached, nil } // Execute runs the command func (c *CommandCommit) Execute(b *Build) (s State, err error) { s = b.state - commits := s.GetCommits() - if commits == "" { + if s.NoCache.IsCached { return s, nil } + commits := s.GetCommits() if s.ImageID == "" && !s.NoBaseImage { return s, fmt.Errorf("Please provide a source image with `from` prior to commit") @@ -1141,7 +1141,7 @@ func (c *CommandMount) Execute(b *Build) (s State, err error) { } s.NoCache.HostConfig.Binds = append(s.NoCache.HostConfig.Binds, - mountsToBinds(c.Mounts)...) + mountsToBinds(c.Mounts, "")...) commitIds = append(commitIds, strings.TrimLeft(c.Name, "/")+":"+arg) } @@ -1192,17 +1192,7 @@ func (c *CommandExport) Execute(b *Build) (s State, err error) { // EXPORT /my/dir /stuff/ --> /EXPORT_VOLUME/stuff/my_dir // EXPORT /my/dir/* / --> /EXPORT_VOLUME/stuff/my_dir - s.Commit("EXPORT %q to %s, prev_export_container: %s", src, dest, b.prevExportContainer) - - name := exportsContainerName(s.ImageID, s.GetCommits()) - if b.currentExportContainerName == "" { - b.currentExportContainerName = name - } - - exportsContainer, err := b.getExportsContainer(b.currentExportContainerName) - if err != nil { - return s, err - } + s.Commit("EXPORT %q to %s, prev_export_container: %s", src, dest, b.prevExportContainerID) // build the command cmdDestPath, err := util.ResolvePath(ExportsPath, dest) @@ -1215,10 +1205,22 @@ func (c *CommandExport) Execute(b *Build) (s State, err error) { return s, err } if hit { - b.prevExportContainer = s.ExportsID + b.prevExportContainerID = s.ExportsID + b.currentExportContainerName = exportsContainerName(s.ImageID, s.GetCommits()) + log.Infof("===EXPORT CONTAINER NAME CACHED: %s ('%s', '%s')", b.currentExportContainerName, s.ImageID, s.GetCommits()) + log.Infof("===CACHED STATE: %v", s) return s, nil } + prevExportContainerName := b.currentExportContainerName + b.currentExportContainerName = exportsContainerName(s.ImageID, s.GetCommits()) + log.Infof("===EXPORT CONTAINER NAME: %s ('%s', '%s')", b.currentExportContainerName, s.ImageID, s.GetCommits()) + + exportsContainer, err := b.getExportsContainerAndSync(b.currentExportContainerName, prevExportContainerName) + if err != nil { + return s, err + } + // Remember original stuff so we can restore it when we finished var exportsID string origState := s @@ -1226,13 +1228,12 @@ func (c *CommandExport) Execute(b *Build) (s State, err error) { defer func() { s = origState s.ExportsID = exportsID - b.prevExportContainer = exportsID + b.prevExportContainerID = exportsID }() // Append exports container as a volume s.NoCache.HostConfig.Binds = append(s.NoCache.HostConfig.Binds, - mountsToBinds(exportsContainer.Mounts)...) - + mountsToBinds(exportsContainer.Mounts, "")...) cmd := []string{"/opt/rsync/bin/rsync", "-a", "--delete-during"} if b.cfg.Verbose { @@ -1259,6 +1260,32 @@ func (c *CommandExport) Execute(b *Build) (s State, err error) { return s, nil } +func (b *Build) getExportsContainerAndSync(currentName, previousName string) (c *docker.Container, err error) { + //If it the first `EXPORT` in the file + //we don't need to sync data from previous container + if previousName == "" { + return b.getExportsContainer(currentName) + } + + prevContainer, err := b.getExportsContainer(previousName) + if err != nil { + return nil, err + } + + binds := mountsToBinds(prevContainer.Mounts, "_source") + + currContainer, err := b.getExportsContainerWithBinds(currentName, binds) + if err != nil { + return nil, err + } + + log.Infof("| Running in %.12s: %s", currContainer.ID, strings.Join(currContainer.Config.Cmd, " ")) + if err = b.client.RunContainer(currContainer.ID, false); err != nil { + return nil, err + } + return currContainer, nil +} + // CommandImport implements IMPORT type CommandImport struct { cfg ConfigCommand @@ -1282,7 +1309,7 @@ func (c *CommandImport) Execute(b *Build) (s State, err error) { if len(args) == 0 { return s, fmt.Errorf("IMPORT requires at least one argument") } - if b.prevExportContainer == "" { + if b.prevExportContainerID == "" { return s, fmt.Errorf("You have to EXPORT something first in order to IMPORT") } @@ -1294,6 +1321,7 @@ func (c *CommandImport) Execute(b *Build) (s State, err error) { if b.currentExportContainerName == "" { return s, fmt.Errorf("You have to EXPORT something first to do IMPORT") } + log.Infof("===IMPORT CONTAINER NAME CACHED: %s", b.currentExportContainerName) exportsContainer, err := b.getExportsContainer(b.currentExportContainerName) if err != nil { @@ -1318,7 +1346,7 @@ func (c *CommandImport) Execute(b *Build) (s State, err error) { src = append(src, argResolved) } - s.Commit("IMPORT %q : %q %s", b.prevExportContainer, src, dest) + s.Commit("IMPORT %q : %q %s", b.prevExportContainerID, src, dest) // Check cache s, hit, err := b.probeCache(s) @@ -1353,7 +1381,7 @@ func (c *CommandImport) Execute(b *Build) (s State, err error) { // Append exports container as a volume s.NoCache.HostConfig.Binds = append(s.NoCache.HostConfig.Binds, - mountsToBinds(exportsContainer.Mounts)...) + mountsToBinds(exportsContainer.Mounts, "")...) if importID, err = b.client.CreateContainer(s); err != nil { return s, err From 7951ab09cae79bf376a985146cc8996906017ab2 Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Wed, 23 Mar 2016 13:36:30 +0000 Subject: [PATCH 21/41] Fixed unit tests --- src/build/build_test.go | 4 ++-- src/build/commands_test.go | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/build/build_test.go b/src/build/build_test.go index 238e9ba3..3db87768 100644 --- a/src/build/build_test.go +++ b/src/build/build_test.go @@ -339,8 +339,8 @@ func (m *MockClient) EnsureImage(imageName string) error { return args.Error(0) } -func (m *MockClient) EnsureContainer(containerName string, config *docker.Config, purpose string) (containerID string, err error) { - args := m.Called(containerName, config, purpose) +func (m *MockClient) EnsureContainer(containerName string, config *docker.Config, hostConfig *docker.HostConfig, purpose string) (containerID string, err error) { + args := m.Called(containerName, config, hostConfig, purpose) return args.String(0), args.Error(1) } diff --git a/src/build/commands_test.go b/src/build/commands_test.go index 0ba42a87..30609791 100644 --- a/src/build/commands_test.go +++ b/src/build/commands_test.go @@ -160,9 +160,10 @@ func TestCommandCommit_NoContainer(t *testing.T) { assert.Equal(t, "", state.NoCache.ContainerID) } -func TestCommandCommit_NoCommitMsgs(t *testing.T) { +func TestCommandCommit_OnCached(t *testing.T) { b, _ := makeBuild(t, "", Config{}) cmd := &CommandCommit{} + b.state = State{NoCache: StateNoCache{IsCached: true}} _, err := cmd.Execute(b) assert.Nil(t, err) @@ -669,7 +670,7 @@ func TestCommandMount_VolumeContainer(t *testing.T) { containerName := b.mountsContainerName("/cache") - c.On("EnsureContainer", containerName, mock.AnythingOfType("*docker.Config"), "/cache").Return("123", nil).Run(func(args mock.Arguments) { + c.On("EnsureContainer", containerName, mock.AnythingOfType("*docker.Config"), mock.AnythingOfType("*docker.HostConfig"), "/cache").Return("123", nil).Run(func(args mock.Arguments) { arg := args.Get(1).(*docker.Config) assert.Equal(t, MountVolumeImage, arg.Image) expectedVolumes := map[string]struct{}{ From 47e398e8bafdbbb7ad2386fde10c7ca6a298f60a Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Wed, 23 Mar 2016 14:03:44 +0000 Subject: [PATCH 22/41] getExportsContainerAndSync moved to build.go --- src/build/build.go | 28 ++++++++++++++++++++++++++++ src/build/commands.go | 26 -------------------------- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/src/build/build.go b/src/build/build.go index f9983e83..b844ff8b 100644 --- a/src/build/build.go +++ b/src/build/build.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/grammarly/rocker/src/imagename" "io" + "strings" "github.com/docker/docker/pkg/units" "github.com/fatih/color" @@ -265,6 +266,33 @@ func (b *Build) getExportsContainerWithBinds(name string, binds []string) (c *do return b.client.InspectContainer(containerID) } + +func (b *Build) getExportsContainerAndSync(currentName, previousName string) (c *docker.Container, err error) { + //If it the first `EXPORT` in the file + //we don't need to sync data from previous container + if previousName == "" { + return b.getExportsContainer(currentName) + } + + prevContainer, err := b.getExportsContainer(previousName) + if err != nil { + return nil, err + } + + binds := mountsToBinds(prevContainer.Mounts, "_source") + + currContainer, err := b.getExportsContainerWithBinds(currentName, binds) + if err != nil { + return nil, err + } + + log.Infof("| Running in %.12s: %s", currContainer.ID, strings.Join(currContainer.Config.Cmd, " ")) + if err = b.client.RunContainer(currContainer.ID, false); err != nil { + return nil, err + } + return currContainer, nil +} + func (b *Build) getExportsContainer(name string) (c *docker.Container, err error) { return b.getExportsContainerWithBinds(name, []string{}) } diff --git a/src/build/commands.go b/src/build/commands.go index 33bf9867..a04f8191 100644 --- a/src/build/commands.go +++ b/src/build/commands.go @@ -1260,32 +1260,6 @@ func (c *CommandExport) Execute(b *Build) (s State, err error) { return s, nil } -func (b *Build) getExportsContainerAndSync(currentName, previousName string) (c *docker.Container, err error) { - //If it the first `EXPORT` in the file - //we don't need to sync data from previous container - if previousName == "" { - return b.getExportsContainer(currentName) - } - - prevContainer, err := b.getExportsContainer(previousName) - if err != nil { - return nil, err - } - - binds := mountsToBinds(prevContainer.Mounts, "_source") - - currContainer, err := b.getExportsContainerWithBinds(currentName, binds) - if err != nil { - return nil, err - } - - log.Infof("| Running in %.12s: %s", currContainer.ID, strings.Join(currContainer.Config.Cmd, " ")) - if err = b.client.RunContainer(currContainer.ID, false); err != nil { - return nil, err - } - return currContainer, nil -} - // CommandImport implements IMPORT type CommandImport struct { cfg ConfigCommand From 9c56018358535a8e635ff9155908b256fcd024d1 Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Wed, 23 Mar 2016 14:05:37 +0000 Subject: [PATCH 23/41] Added Roman Khlystik to CONTRIBUTORS file --- CONTRIBUTORS | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index cebf8a55..9d67abee 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -4,3 +4,4 @@ Pavel Forkert Stas Levental Vsevolod Polyakov Yuriy Bogdanov +Roman Khlystik From 2a21a2d9e434957ba2a092068d17142e00a6dab1 Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Wed, 23 Mar 2016 14:06:22 +0000 Subject: [PATCH 24/41] Log messages with EXPORT contaner name stuff moved to Debug level --- src/build/commands.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/build/commands.go b/src/build/commands.go index a04f8191..f367d340 100644 --- a/src/build/commands.go +++ b/src/build/commands.go @@ -1207,14 +1207,13 @@ func (c *CommandExport) Execute(b *Build) (s State, err error) { if hit { b.prevExportContainerID = s.ExportsID b.currentExportContainerName = exportsContainerName(s.ImageID, s.GetCommits()) - log.Infof("===EXPORT CONTAINER NAME CACHED: %s ('%s', '%s')", b.currentExportContainerName, s.ImageID, s.GetCommits()) - log.Infof("===CACHED STATE: %v", s) + log.Debugf("===EXPORT CONTAINER NAME CACHED: %s ('%s', '%s')", b.currentExportContainerName, s.ImageID, s.GetCommits()) return s, nil } prevExportContainerName := b.currentExportContainerName b.currentExportContainerName = exportsContainerName(s.ImageID, s.GetCommits()) - log.Infof("===EXPORT CONTAINER NAME: %s ('%s', '%s')", b.currentExportContainerName, s.ImageID, s.GetCommits()) + log.Debugf("===EXPORT CONTAINER NAME: %s ('%s', '%s')", b.currentExportContainerName, s.ImageID, s.GetCommits()) exportsContainer, err := b.getExportsContainerAndSync(b.currentExportContainerName, prevExportContainerName) if err != nil { @@ -1295,7 +1294,7 @@ func (c *CommandImport) Execute(b *Build) (s State, err error) { if b.currentExportContainerName == "" { return s, fmt.Errorf("You have to EXPORT something first to do IMPORT") } - log.Infof("===IMPORT CONTAINER NAME CACHED: %s", b.currentExportContainerName) + log.Debugf("===IMPORT CONTAINER NAME CACHED: %s", b.currentExportContainerName) exportsContainer, err := b.getExportsContainer(b.currentExportContainerName) if err != nil { From f403206005220ecf5eda4ea0d22a8ad924f188f1 Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Wed, 23 Mar 2016 15:00:35 +0000 Subject: [PATCH 25/41] Added TestDoubleExport --- test/export_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/export_test.go b/test/export_test.go index 0afc6b4a..08d3a52e 100644 --- a/test/export_test.go +++ b/test/export_test.go @@ -203,3 +203,19 @@ func TestExportSameFileFewFroms(t *testing.T) { assert.Nil(t, err, "Can't read file") assert.Equal(t, "first_few", string(content)) } + +func TestDoubleExport(t *testing.T) { + rockerContent := `FROM alpine + EXPORT /etc/issue issue + EXPORT /etc/hostname hostname + + FROM alpine + IMPORT issue + IMPORT hostname` + + err := runRockerBuildWithOptions(rockerContent, "--reload-cache") + assert.Nil(t, err) + + err = runRockerBuildWithOptions(rockerContent) + assert.Nil(t, err) +} From fa17eda958d0b0307c1f98d89fc6d452c5b19592 Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Wed, 23 Mar 2016 15:02:34 +0000 Subject: [PATCH 26/41] Will clean commits and use ParentID in case of cache hit --- src/build/build.go | 2 +- src/build/commands.go | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/build/build.go b/src/build/build.go index b844ff8b..08e82638 100644 --- a/src/build/build.go +++ b/src/build/build.go @@ -286,7 +286,7 @@ func (b *Build) getExportsContainerAndSync(currentName, previousName string) (c return nil, err } - log.Infof("| Running in %.12s: %s", currContainer.ID, strings.Join(currContainer.Config.Cmd, " ")) + log.Infof("| Running in %s: %s", currentName, strings.Join(currContainer.Config.Cmd, " ")) if err = b.client.RunContainer(currContainer.ID, false); err != nil { return nil, err } diff --git a/src/build/commands.go b/src/build/commands.go index f367d340..e1420886 100644 --- a/src/build/commands.go +++ b/src/build/commands.go @@ -1206,14 +1206,16 @@ func (c *CommandExport) Execute(b *Build) (s State, err error) { } if hit { b.prevExportContainerID = s.ExportsID - b.currentExportContainerName = exportsContainerName(s.ImageID, s.GetCommits()) - log.Debugf("===EXPORT CONTAINER NAME CACHED: %s ('%s', '%s')", b.currentExportContainerName, s.ImageID, s.GetCommits()) + b.currentExportContainerName = exportsContainerName(s.ParentID, s.GetCommits()) + log.Infof("| Export container: %s", b.currentExportContainerName) + log.Debugf("===EXPORT CONTAINER NAME: %s ('%s', '%s')", b.currentExportContainerName, s.ParentID, s.GetCommits()) + + s.CleanCommits() return s, nil } prevExportContainerName := b.currentExportContainerName b.currentExportContainerName = exportsContainerName(s.ImageID, s.GetCommits()) - log.Debugf("===EXPORT CONTAINER NAME: %s ('%s', '%s')", b.currentExportContainerName, s.ImageID, s.GetCommits()) exportsContainer, err := b.getExportsContainerAndSync(b.currentExportContainerName, prevExportContainerName) if err != nil { From d67ddf91f667ee692d5222c9126eccc25657d999 Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Wed, 23 Mar 2016 16:11:55 +0000 Subject: [PATCH 27/41] Added more cache tests --- test/cache_test.go | 95 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/test/cache_test.go b/test/cache_test.go index 3486663b..5c994c20 100644 --- a/test/cache_test.go +++ b/test/cache_test.go @@ -1,7 +1,12 @@ package tests import ( + "github.com/stretchr/testify/assert" + "io/ioutil" + "os" + "strconv" "testing" + "time" ) func TestCacheWithEnvVariables(t *testing.T) { @@ -109,3 +114,93 @@ TAG `+tag, "--no-cache") t.Fatalf("Sha of images are equal but shouldn't. sha1: %s, sha2: %s", sha1, sha2) } } + +func TestCacheFewSameCommands(t *testing.T) { + tag := "rocker-integratin-test:1.8.35" + defer removeImage(tag) + scenario := `FROM alpine + RUN true + RUN true + RUN true + RUN true + RUN true + TAG ` + tag + + err := runRockerBuildWithOptions(scenario, "--reload-cache") + assert.Nil(t, err) + sha1, err := getImageShaByName(tag) + assert.Nil(t, err) + + err = runRockerBuildWithOptions(scenario) + assert.Nil(t, err) + sha2, err := getImageShaByName(tag) + assert.Nil(t, err) + + assert.Equal(t, sha1, sha2) +} + +func TestCacheFewDifferentCommands(t *testing.T) { + tag := "rocker-integratin-test:1.8.36" + defer removeImage(tag) + scenario := `FROM alpine + RUN true + RUN echo "123" > /foobar + RUN ls > /dev/null + RUN date + TAG ` + tag + + err := runRockerBuildWithOptions(scenario, "--reload-cache") + assert.Nil(t, err) + sha1, err := getImageShaByName(tag) + assert.Nil(t, err) + + err = runRockerBuildWithOptions(scenario) + assert.Nil(t, err) + sha2, err := getImageShaByName(tag) + assert.Nil(t, err) + + assert.Equal(t, sha1, sha2) +} + +func TestCacheAndExportImport(t *testing.T) { + tagExport := "rocker-integratin-test:export" + tagImport := "rocker-integratin-test:import" + defer removeImage(tagExport) + defer removeImage(tagImport) + + dir, err := ioutil.TempDir("/tmp", "rocker_integration_test_export_") + assert.Nil(t, err, "Can't create tmp dir") + defer os.RemoveAll(dir) + + randomData := strconv.Itoa(int(time.Now().UnixNano() % int64(100000001))) + scenario := `FROM alpine + RUN true + RUN echo -n ` + randomData + ` > /foobar + EXPORT /foobar + TAG ` + tagExport + ` + FROM alpine + MOUNT ` + dir + `:/datadir + IMPORT /foobar /datadir/foobar + TAG ` + tagImport + + err = runRockerBuildWithOptions(scenario, "--reload-cache") + assert.Nil(t, err) + shaExport1, err := getImageShaByName(tagExport) + assert.Nil(t, err) + shaImport1, err := getImageShaByName(tagImport) + assert.Nil(t, err) + content, err := ioutil.ReadFile(dir + "/foobar") + assert.Equal(t, string(content), randomData) + + err = runRockerBuildWithOptions(scenario, "--reload-cache") + assert.Nil(t, err) + shaExport2, err := getImageShaByName(tagExport) + assert.Nil(t, err) + shaImport2, err := getImageShaByName(tagImport) + assert.Nil(t, err) + content, err = ioutil.ReadFile(dir + "/foobar") + assert.Equal(t, string(content), randomData) + + assert.Equal(t, shaExport1, shaExport2) + assert.Equal(t, shaImport1, shaImport2) +} From d03b5a7524f47a75f39b6ce03178f9ae838717f6 Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Wed, 23 Mar 2016 16:35:27 +0000 Subject: [PATCH 28/41] Fixed test --- test/cache_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cache_test.go b/test/cache_test.go index 5c994c20..f1dc705e 100644 --- a/test/cache_test.go +++ b/test/cache_test.go @@ -192,7 +192,7 @@ func TestCacheAndExportImport(t *testing.T) { content, err := ioutil.ReadFile(dir + "/foobar") assert.Equal(t, string(content), randomData) - err = runRockerBuildWithOptions(scenario, "--reload-cache") + err = runRockerBuildWithOptions(scenario) assert.Nil(t, err) shaExport2, err := getImageShaByName(tagExport) assert.Nil(t, err) From dc3bd8eeda86808ecf02c8d0bda472c436e34ab7 Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Wed, 23 Mar 2016 16:40:53 +0000 Subject: [PATCH 29/41] Clean commits in Commit command --- src/build/commands.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/build/commands.go b/src/build/commands.go index e1420886..c0c5eeac 100644 --- a/src/build/commands.go +++ b/src/build/commands.go @@ -277,6 +277,9 @@ func (c *CommandCommit) String() string { // ShouldRun returns true if the command should be executed func (c *CommandCommit) ShouldRun(b *Build) (bool, error) { + if b.state.NoCache.IsCached { + b.state.CleanCommits() + } return !b.state.NoCache.IsCached, nil } @@ -285,6 +288,7 @@ func (c *CommandCommit) Execute(b *Build) (s State, err error) { s = b.state if s.NoCache.IsCached { + s.CleanCommits() return s, nil } commits := s.GetCommits() @@ -1209,8 +1213,6 @@ func (c *CommandExport) Execute(b *Build) (s State, err error) { b.currentExportContainerName = exportsContainerName(s.ParentID, s.GetCommits()) log.Infof("| Export container: %s", b.currentExportContainerName) log.Debugf("===EXPORT CONTAINER NAME: %s ('%s', '%s')", b.currentExportContainerName, s.ParentID, s.GetCommits()) - - s.CleanCommits() return s, nil } From 12ef9cc3f9d5f25015921a4f9cb9398859a1e447 Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Wed, 23 Mar 2016 16:41:22 +0000 Subject: [PATCH 30/41] Added description of fails --- test/cache_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/cache_test.go b/test/cache_test.go index f1dc705e..87afe5a6 100644 --- a/test/cache_test.go +++ b/test/cache_test.go @@ -201,6 +201,6 @@ func TestCacheAndExportImport(t *testing.T) { content, err = ioutil.ReadFile(dir + "/foobar") assert.Equal(t, string(content), randomData) - assert.Equal(t, shaExport1, shaExport2) - assert.Equal(t, shaImport1, shaImport2) + assert.Equal(t, shaExport1, shaExport2, "Export doesn't match") + assert.Equal(t, shaImport1, shaImport2, "Import doesn't match") } From 2ffaba4b5bfde6c6d2d18b2440fa33ace4928bbb Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Thu, 24 Mar 2016 10:35:13 +0000 Subject: [PATCH 31/41] Added TestCacheMountNotCached --- test/cache_test.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/cache_test.go b/test/cache_test.go index 87afe5a6..c5c24829 100644 --- a/test/cache_test.go +++ b/test/cache_test.go @@ -162,6 +162,28 @@ func TestCacheFewDifferentCommands(t *testing.T) { assert.Equal(t, sha1, sha2) } +func TestCacheMountNotCached(t *testing.T) { + tag := "rocker-integratin-test:mount_not_cached" + defer removeImage(tag) + scenario1 := `FROM alpine + TAG ` + tag + scenario2 := `FROM alpine + MOUNT /tmp:/tmp + TAG ` + tag + + err := runRockerBuildWithOptions(scenario1, "--reload-cache") + assert.Nil(t, err) + sha1, err := getImageShaByName(tag) + assert.Nil(t, err) + + err = runRockerBuildWithOptions(scenario2) + assert.Nil(t, err) + sha2, err := getImageShaByName(tag) + assert.Nil(t, err) + + assert.Equal(t, sha1, sha2) +} + func TestCacheAndExportImport(t *testing.T) { tagExport := "rocker-integratin-test:export" tagImport := "rocker-integratin-test:import" From 24491e26482aabaef5141f6f1c73c3da7098adaf Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Thu, 24 Mar 2016 11:31:56 +0000 Subject: [PATCH 32/41] Changed MOUNT cached test --- test/cache_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/cache_test.go b/test/cache_test.go index c5c24829..5adb714a 100644 --- a/test/cache_test.go +++ b/test/cache_test.go @@ -181,7 +181,8 @@ func TestCacheMountNotCached(t *testing.T) { sha2, err := getImageShaByName(tag) assert.Nil(t, err) - assert.Equal(t, sha1, sha2) + //Despite the `MOUNT` isn't "commited" command it still will invalidate the cache + assert.NotEqual(t, sha1, sha2) } func TestCacheAndExportImport(t *testing.T) { From 11e8f59c3e47b30bc39b3c5e2b2be38c93b797fa Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Thu, 24 Mar 2016 11:33:03 +0000 Subject: [PATCH 33/41] Get rid of IsCached. Returned back to GetCommits logic --- src/build/build.go | 12 ++++++++---- src/build/commands.go | 16 +++++++--------- src/build/state.go | 1 - 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/build/build.go b/src/build/build.go index 08e82638..c80d24c1 100644 --- a/src/build/build.go +++ b/src/build/build.go @@ -157,7 +157,14 @@ func (b *Build) GetImageID() string { } func (b *Build) probeCache(s State) (cachedState State, hit bool, err error) { - s.NoCache.IsCached = false + cachedState, hit, err = b.probeCacheAndPreserveCommits(s) + if hit && err == nil { + cachedState.CleanCommits() + } + return +} + +func (b *Build) probeCacheAndPreserveCommits(s State) (cachedState State, hit bool, err error) { if b.cache == nil || s.NoCache.CacheBusted { return s, false, nil @@ -207,9 +214,6 @@ func (b *Build) probeCache(s State) (cachedState State, hit bool, err error) { // Keep items that should not be cached from the previous state s2.NoCache = s.NoCache - // Mark state as cached - s2.NoCache.IsCached = true - return *s2, true, nil } diff --git a/src/build/commands.go b/src/build/commands.go index c0c5eeac..1781ab01 100644 --- a/src/build/commands.go +++ b/src/build/commands.go @@ -277,21 +277,17 @@ func (c *CommandCommit) String() string { // ShouldRun returns true if the command should be executed func (c *CommandCommit) ShouldRun(b *Build) (bool, error) { - if b.state.NoCache.IsCached { - b.state.CleanCommits() - } - return !b.state.NoCache.IsCached, nil + return b.state.GetCommits() != "", nil } // Execute runs the command func (c *CommandCommit) Execute(b *Build) (s State, err error) { s = b.state - if s.NoCache.IsCached { - s.CleanCommits() + commits := s.GetCommits() + if commits == "" { return s, nil } - commits := s.GetCommits() if s.ImageID == "" && !s.NoBaseImage { return s, fmt.Errorf("Please provide a source image with `from` prior to commit") @@ -1196,7 +1192,7 @@ func (c *CommandExport) Execute(b *Build) (s State, err error) { // EXPORT /my/dir /stuff/ --> /EXPORT_VOLUME/stuff/my_dir // EXPORT /my/dir/* / --> /EXPORT_VOLUME/stuff/my_dir - s.Commit("EXPORT %q to %s, prev_export_container: %s", src, dest, b.prevExportContainerID) + s.Commit("EXPORT %q to %s, prev_export_container_salt: %s", src, dest, b.prevExportContainerID) // build the command cmdDestPath, err := util.ResolvePath(ExportsPath, dest) @@ -1204,7 +1200,7 @@ func (c *CommandExport) Execute(b *Build) (s State, err error) { return s, fmt.Errorf("Invalid EXPORT destination: %s", dest) } - s, hit, err := b.probeCache(s) + s, hit, err := b.probeCacheAndPreserveCommits(s) if err != nil { return s, err } @@ -1213,6 +1209,7 @@ func (c *CommandExport) Execute(b *Build) (s State, err error) { b.currentExportContainerName = exportsContainerName(s.ParentID, s.GetCommits()) log.Infof("| Export container: %s", b.currentExportContainerName) log.Debugf("===EXPORT CONTAINER NAME: %s ('%s', '%s')", b.currentExportContainerName, s.ParentID, s.GetCommits()) + s.CleanCommits() return s, nil } @@ -1324,6 +1321,7 @@ func (c *CommandImport) Execute(b *Build) (s State, err error) { } s.Commit("IMPORT %q : %q %s", b.prevExportContainerID, src, dest) + log.Infof("===IMPORT: %q", s.GetCommits()) // Check cache s, hit, err := b.probeCache(s) diff --git a/src/build/state.go b/src/build/state.go index 518d901a..3fcf1085 100644 --- a/src/build/state.go +++ b/src/build/state.go @@ -46,7 +46,6 @@ type StateNoCache struct { CmdSet bool ContainerID string HostConfig docker.HostConfig - IsCached bool } // NewState makes a fresh state From 47014a382bf58fed460b5d85335fa2e38331507c Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Thu, 24 Mar 2016 12:25:53 +0000 Subject: [PATCH 34/41] Added verbosity command line option to `go test` --- test/init_test.go | 16 ++++++++++++++++ test/utils.go | 10 ++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 test/init_test.go diff --git a/test/init_test.go b/test/init_test.go new file mode 100644 index 00000000..83479e09 --- /dev/null +++ b/test/init_test.go @@ -0,0 +1,16 @@ +package tests + +import ( + "flag" + "os" + "testing" +) + +var ( + verbosity_level = flag.Int("verbosity", 0, "Level of verbosity. 0 - nothing, 1 - cmds, 2 - output of cmd's") +) + +func TestMain(m *testing.M) { + flag.Parse() + os.Exit(m.Run()) +} diff --git a/test/utils.go b/test/utils.go index 9be728b4..78294266 100644 --- a/test/utils.go +++ b/test/utils.go @@ -23,11 +23,16 @@ func getGOPATH() string { func runCmd(executable string, stdoutWriter io.Writer /* stderr io.Writer,*/, params ...string) error { cmd := exec.Command(executable, params...) - fmt.Printf("Running: %v\n", strings.Join(cmd.Args, " ")) + if *verbosity_level >= 1 { + fmt.Printf("Running: %v\n", strings.Join(cmd.Args, " ")) + } + if stdoutWriter != nil { cmd.Stdout = stdoutWriter + } else if *verbosity_level >= 2 { + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr } - //cmd.Stderr = stderr if err := cmd.Run(); err != nil { //fmt.Printf("Failed to run '%v' with arguments '%v'\n", executable, params) @@ -94,6 +99,7 @@ func runRockerBuildWithFile(filename string, opts ...string) error { p := []string{"build", "-f", filename} params := append(p, opts...) + if err := runCmd(gopath+"/bin/rocker", nil, params...); err != nil { return err } From 7bc4036dd1b8205b79d36274e25473aeaab0732e Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Thu, 24 Mar 2016 13:39:39 +0000 Subject: [PATCH 35/41] Will invalidate before import in TestExportSeparateFilesSameExport --- test/export_test.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/test/export_test.go b/test/export_test.go index 08d3a52e..a4fc7ef7 100644 --- a/test/export_test.go +++ b/test/export_test.go @@ -66,7 +66,7 @@ func TestExportSeparateFilesDifferentExport(t *testing.T) { } func TestExportSeparateFilesSameExport(t *testing.T) { - dir, err := ioutil.TempDir("/tmp", "rocker_integration_test_export_") + dir, err := ioutil.TempDir("/tmp", "rocker_integration_test_export_sep") assert.Nil(t, err, "Can't create tmp dir") defer os.RemoveAll(dir) @@ -77,7 +77,8 @@ func TestExportSeparateFilesSameExport(t *testing.T) { FROM alpine MOUNT ` + dir + `:/datadir - IMPORT /exported_file /datadir/imported_file` + IMPORT /exported_file /datadir/imported_file + ` rockerContentSecond := `FROM alpine:latest EXPORT /etc/issue @@ -88,13 +89,24 @@ func TestExportSeparateFilesSameExport(t *testing.T) { MOUNT ` + dir + `:/datadir IMPORT /exported_file /datadir/imported_file` + rockerContentThird := `FROM alpine:latest + EXPORT /etc/issue + RUN echo -n "first_separate" > /exported_file + EXPORT /exported_file + + FROM alpine + MOUNT ` + dir + `:/datadir + RUN echo "Invalidate cache" + IMPORT /exported_file /datadir/imported_file + ` + err = runRockerBuildWithOptions(rockerContentFirst, "--reload-cache") assert.Nil(t, err) err = runRockerBuildWithOptions(rockerContentSecond) assert.Nil(t, err) - err = runRockerBuildWithOptions(rockerContentFirst) + err = runRockerBuildWithOptions(rockerContentThird) assert.Nil(t, err) content, err := ioutil.ReadFile(dir + "/imported_file") From 99eb51ed0b2b6220e6c25b2043dd9d21263d8cf4 Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Thu, 24 Mar 2016 13:41:54 +0000 Subject: [PATCH 36/41] make lint happy --- test/init_test.go | 2 +- test/utils.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/init_test.go b/test/init_test.go index 83479e09..6eefc069 100644 --- a/test/init_test.go +++ b/test/init_test.go @@ -7,7 +7,7 @@ import ( ) var ( - verbosity_level = flag.Int("verbosity", 0, "Level of verbosity. 0 - nothing, 1 - cmds, 2 - output of cmd's") + verbosityLevel = flag.Int("verbosity", 0, "Level of verbosity. 0 - nothing, 1 - cmds, 2 - output of cmd's") ) func TestMain(m *testing.M) { diff --git a/test/utils.go b/test/utils.go index 78294266..90ccfdc2 100644 --- a/test/utils.go +++ b/test/utils.go @@ -23,13 +23,13 @@ func getGOPATH() string { func runCmd(executable string, stdoutWriter io.Writer /* stderr io.Writer,*/, params ...string) error { cmd := exec.Command(executable, params...) - if *verbosity_level >= 1 { + if *verbosityLevel >= 1 { fmt.Printf("Running: %v\n", strings.Join(cmd.Args, " ")) } if stdoutWriter != nil { cmd.Stdout = stdoutWriter - } else if *verbosity_level >= 2 { + } else if *verbosityLevel >= 2 { cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr } From c3ed1d895bd5a161e135ff060e72ab6f347f2e39 Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Thu, 24 Mar 2016 13:45:00 +0000 Subject: [PATCH 37/41] Fixed unit test --- src/build/commands_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/build/commands_test.go b/src/build/commands_test.go index 30609791..2fc24f82 100644 --- a/src/build/commands_test.go +++ b/src/build/commands_test.go @@ -160,10 +160,9 @@ func TestCommandCommit_NoContainer(t *testing.T) { assert.Equal(t, "", state.NoCache.ContainerID) } -func TestCommandCommit_OnCached(t *testing.T) { +func TestCommandCommit_NoCommitMsgs(t *testing.T) { b, _ := makeBuild(t, "", Config{}) cmd := &CommandCommit{} - b.state = State{NoCache: StateNoCache{IsCached: true}} _, err := cmd.Execute(b) assert.Nil(t, err) From 78f3bea64359d00b885155121d91f99c03b6052d Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Thu, 24 Mar 2016 14:16:36 +0000 Subject: [PATCH 38/41] Removed extra debug for import --- src/build/commands.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/build/commands.go b/src/build/commands.go index 1781ab01..18c12473 100644 --- a/src/build/commands.go +++ b/src/build/commands.go @@ -1295,7 +1295,6 @@ func (c *CommandImport) Execute(b *Build) (s State, err error) { if b.currentExportContainerName == "" { return s, fmt.Errorf("You have to EXPORT something first to do IMPORT") } - log.Debugf("===IMPORT CONTAINER NAME CACHED: %s", b.currentExportContainerName) exportsContainer, err := b.getExportsContainer(b.currentExportContainerName) if err != nil { @@ -1321,7 +1320,6 @@ func (c *CommandImport) Execute(b *Build) (s State, err error) { } s.Commit("IMPORT %q : %q %s", b.prevExportContainerID, src, dest) - log.Infof("===IMPORT: %q", s.GetCommits()) // Check cache s, hit, err := b.probeCache(s) From 6d42a28281409647919fb9eb155cd8b7eceea35d Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Thu, 24 Mar 2016 14:21:02 +0000 Subject: [PATCH 39/41] Integration test output would be verbose --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 90c14b64..ec1bc31c 100644 --- a/Makefile +++ b/Makefile @@ -65,6 +65,6 @@ test: testdeps fmtcheck vet lint GO15VENDOREXPERIMENT=1 go test ./src/... $(TESTARGS) test_integration: - GO15VENDOREXPERIMENT=1 go test ./test/... + GO15VENDOREXPERIMENT=1 go test -v ./test/... --args verbosity=0 .PHONY: clean test fmtcheck lint vet gocyclo default From e93315351b961d7967a907b72ed31abf4b47a42d Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Thu, 24 Mar 2016 14:59:47 +0000 Subject: [PATCH 40/41] Added TestExportSmolinIssue test --- test/export_test.go | 70 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 7 deletions(-) diff --git a/test/export_test.go b/test/export_test.go index a4fc7ef7..57e65042 100644 --- a/test/export_test.go +++ b/test/export_test.go @@ -65,6 +65,62 @@ func TestExportSeparateFilesDifferentExport(t *testing.T) { assert.Equal(t, "first_diff", string(content)) } +func TestExportSmolinIssue(t *testing.T) { + tag := "rocker-integratin-test-export-smolin" + defer removeImage(tag + ":qa") + defer removeImage(tag + ":prod") + + dir, err := ioutil.TempDir("/tmp", "rocker_integration_test_export_smolin") + assert.Nil(t, err, "Can't create tmp dir") + defer os.RemoveAll(dir) + + rockerfile, err := createTempFile("") + assert.Nil(t, err, "Can't create temp file") + defer os.RemoveAll(rockerfile) + + rockerContentFirst := []byte(` {{ $env := .env}} + FROM alpine + RUN echo -n "{{ $env }}" > /exported_file + EXPORT /exported_file + + FROM alpine + IMPORT /exported_file /imported_file + TAG ` + tag + ":{{ $env }}") + + err = ioutil.WriteFile(rockerfile, rockerContentFirst, 0644) + assert.Nil(t, err) + err = runRockerBuildWithFile(rockerfile, "--reload-cache", "--var", "env=qa") + assert.Nil(t, err) + + err = ioutil.WriteFile(rockerfile, rockerContentFirst, 0644) + assert.Nil(t, err) + err = runRockerBuildWithFile(rockerfile, "--var", "env=prod") + assert.Nil(t, err) + + err = ioutil.WriteFile(rockerfile, rockerContentFirst, 0644) + assert.Nil(t, err) + err = runRockerBuildWithFile(rockerfile, "--var", "env=qa") + assert.Nil(t, err) + + content := `FROM ` + tag + `:qa + MOUNT ` + dir + `:/data + RUN cp /imported_file /data/qa.file + + FROM ` + tag + `:prod + MOUNT ` + dir + `:/data + RUN cp /imported_file /data/prod.file` + err = runRockerBuildWithOptions(content, "--no-cache") + assert.Nil(t, err) + + qaContent, err := ioutil.ReadFile(dir + "/qa.file") + assert.Nil(t, err) + assert.Equal(t, string(qaContent), "qa") + + prodContent, err := ioutil.ReadFile(dir + "/prod.file") + assert.Nil(t, err) + assert.Equal(t, string(prodContent), "prod") + +} func TestExportSeparateFilesSameExport(t *testing.T) { dir, err := ioutil.TempDir("/tmp", "rocker_integration_test_export_sep") assert.Nil(t, err, "Can't create tmp dir") @@ -114,7 +170,7 @@ func TestExportSeparateFilesSameExport(t *testing.T) { assert.Equal(t, "first_separate", string(content)) } -func TestExportSmolinIssue(t *testing.T) { +func TestExportSameFileDifferentCmd(t *testing.T) { dir, err := ioutil.TempDir("/tmp", "rocker_integration_test_export_") assert.Nil(t, err, "Can't create tmp dir") defer os.RemoveAll(dir) @@ -124,21 +180,21 @@ func TestExportSmolinIssue(t *testing.T) { defer os.RemoveAll(rockerfile) rockerContentFirst := []byte(`FROM alpine - RUN echo -n "first_smolin" > /exported_file + RUN echo -n "first_foobar1" > /exported_file EXPORT /exported_file FROM alpine MOUNT ` + dir + `:/datadir IMPORT /exported_file /datadir/imported_file`) rockerContentSecond := []byte(`FROM alpine - RUN echo -n "second_smolin" > /exported_file + RUN echo -n "second_foobar1" > /exported_file EXPORT /exported_file FROM alpine MOUNT ` + dir + `:/datadir IMPORT /exported_file /datadir/imported_file`) rockerContentThird := []byte(`FROM alpine - RUN echo -n "first_smolin" > /exported_file + RUN echo -n "first_foobar1" > /exported_file EXPORT /exported_file FROM alpine MOUNT ` + dir + `:/datadir @@ -151,7 +207,7 @@ func TestExportSmolinIssue(t *testing.T) { assert.Nil(t, err) content, err := ioutil.ReadFile(dir + "/imported_file") assert.Nil(t, err) - assert.Equal(t, "first_smolin", string(content)) + assert.Equal(t, "first_foobar1", string(content)) err = ioutil.WriteFile(rockerfile, rockerContentSecond, 0644) assert.Nil(t, err) @@ -159,7 +215,7 @@ func TestExportSmolinIssue(t *testing.T) { assert.Nil(t, err) content, err = ioutil.ReadFile(dir + "/imported_file") assert.Nil(t, err) - assert.Equal(t, "second_smolin", string(content)) + assert.Equal(t, "second_foobar1", string(content)) err = ioutil.WriteFile(rockerfile, rockerContentThird, 0644) assert.Nil(t, err) @@ -167,7 +223,7 @@ func TestExportSmolinIssue(t *testing.T) { assert.Nil(t, err) content, err = ioutil.ReadFile(dir + "/imported_file") assert.Nil(t, err, "Can't read file") - assert.Equal(t, "first_smolin", string(content)) + assert.Equal(t, "first_foobar1", string(content)) } func TestExportSameFileFewFroms(t *testing.T) { From 0947c23946e358ad7158277e4bfc1ee32bc7bb99 Mon Sep 17 00:00:00 2001 From: Roman Khlystik Date: Thu, 24 Mar 2016 15:25:47 +0000 Subject: [PATCH 41/41] Added cache invalidation to be able to make IMPORT again --- test/export_test.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/test/export_test.go b/test/export_test.go index 57e65042..6368c040 100644 --- a/test/export_test.go +++ b/test/export_test.go @@ -4,7 +4,9 @@ import ( "github.com/stretchr/testify/assert" "io/ioutil" "os" + "strconv" "testing" + "time" ) func TestExportSimple(t *testing.T) { @@ -77,6 +79,7 @@ func TestExportSmolinIssue(t *testing.T) { rockerfile, err := createTempFile("") assert.Nil(t, err, "Can't create temp file") defer os.RemoveAll(rockerfile) + randomData := strconv.Itoa(int(time.Now().UnixNano() % int64(100000001))) rockerContentFirst := []byte(` {{ $env := .env}} FROM alpine @@ -87,6 +90,16 @@ func TestExportSmolinIssue(t *testing.T) { IMPORT /exported_file /imported_file TAG ` + tag + ":{{ $env }}") + rockerContentSecond := []byte(` {{ $env := .env}} + FROM alpine + RUN echo -n "{{ $env }}" > /exported_file + EXPORT /exported_file + + FROM alpine + RUN echo "invalidate with ` + randomData + `" + IMPORT /exported_file /imported_file + TAG ` + tag + ":{{ $env }}") + err = ioutil.WriteFile(rockerfile, rockerContentFirst, 0644) assert.Nil(t, err) err = runRockerBuildWithFile(rockerfile, "--reload-cache", "--var", "env=qa") @@ -94,10 +107,10 @@ func TestExportSmolinIssue(t *testing.T) { err = ioutil.WriteFile(rockerfile, rockerContentFirst, 0644) assert.Nil(t, err) - err = runRockerBuildWithFile(rockerfile, "--var", "env=prod") + err = runRockerBuildWithFile(rockerfile, "--reload-cache", "--var", "env=prod") assert.Nil(t, err) - err = ioutil.WriteFile(rockerfile, rockerContentFirst, 0644) + err = ioutil.WriteFile(rockerfile, rockerContentSecond, 0644) assert.Nil(t, err) err = runRockerBuildWithFile(rockerfile, "--var", "env=qa") assert.Nil(t, err)