Fixed issue with export/import

CONTRIBUTORS

@@ -4,3 +4,4 @@ Pavel Forkert <fxposter@gmail.com>
 Stas Levental <stanislav.levental@grammarly.com>
 Vsevolod Polyakov <vsevolod.polyakov@grammarly.com>
 Yuriy Bogdanov <yuriy.bogdanov@grammarly.com>
+Roman Khlystik <roman.khlystik@grammarly.com>

Makefile

@@ -65,6 +65,6 @@ test: testdeps fmtcheck vet lint
 	GO15VENDOREXPERIMENT=1 go test ./src/... $(TESTARGS)
 
 test_integration:
-	GO15VENDOREXPERIMENT=1 go test ./test/...
+	GO15VENDOREXPERIMENT=1 go test -v ./test/... --args verbosity=0
 
 .PHONY: clean test fmtcheck lint vet gocyclo default

src/build/build.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"github.com/grammarly/rocker/src/imagename"
 	"io"
+	"strings"
 
 	"github.com/docker/docker/pkg/units"
 	"github.com/fatih/color"
@@ -75,6 +76,9 @@ type Build struct {
 	// A little hack to support cross-FROM cache for EXPORTS
 	// maybe rethink it later
 	exports []string
+
+	currentExportContainerName string
+	prevExportContainerID      string
 }
 
 // New creates the new build object
@@ -94,26 +98,26 @@ func New(client Client, rockerfile *Rockerfile, cache Cache, cfg Config) *Build
 func (b *Build) Run(plan Plan) (err error) {
 
 	for k := 0; k < len(plan); k++ {
-		c := plan[k]
+		command := plan[k]
 
-		log.Debugf("Step %d: %# v", k+1, pretty.Formatter(c))
+		log.Debugf("Step %d: %# v", k+1, pretty.Formatter(command))
 
 		var doRun bool
-		if doRun, err = c.ShouldRun(b); err != nil {
+		if doRun, err = command.ShouldRun(b); err != nil {
 			return err
 		}
 		if !doRun {
 			continue
 		}
 
 		// Replace env for the command if appropriate
-		if c, ok := c.(EnvReplacableCommand); ok {
-			c.ReplaceEnv(b.state.Config.Env)
+		if command, ok := command.(EnvReplacableCommand); ok {
+			command.ReplaceEnv(b.state.Config.Env)
 		}
 
-		log.Infof("%s", color.New(color.FgWhite, color.Bold).SprintFunc()(c))
+		log.Infof("%s", color.New(color.FgWhite, color.Bold).SprintFunc()(command))
 
-		if b.state, err = c.Execute(b); err != nil {
+		if b.state, err = command.Execute(b); err != nil {
 			return err
 		}
 
@@ -153,6 +157,15 @@ func (b *Build) GetImageID() string {
 }
 
 func (b *Build) probeCache(s State) (cachedState State, hit bool, err error) {
+	cachedState, hit, err = b.probeCacheAndPreserveCommits(s)
+	if hit && err == nil {
+		cachedState.CleanCommits()
+	}
+	return
+}
+
+func (b *Build) probeCacheAndPreserveCommits(s State) (cachedState State, hit bool, err error) {
+
 	if b.cache == nil || s.NoCache.CacheBusted {
 		return s, false, nil
 	}
@@ -200,8 +213,6 @@ func (b *Build) probeCache(s State) (cachedState State, hit bool, err error) {
 
 	// Keep items that should not be cached from the previous state
 	s2.NoCache = s.NoCache
-	// We don't want commits to go through the cache
-	s2.CleanCommits()
 
 	return *s2, true, nil
 }
@@ -219,7 +230,7 @@ func (b *Build) getVolumeContainer(path string) (c *docker.Container, err error)
 
 	log.Debugf("Make MOUNT volume container %s with options %# v", name, config)
 
-	if _, err = b.client.EnsureContainer(name, config, path); err != nil {
+	if _, err = b.client.EnsureContainer(name, config, nil, path); err != nil {
 		return nil, err
 	}
 
@@ -228,20 +239,29 @@ func (b *Build) getVolumeContainer(path string) (c *docker.Container, err error)
 	return b.client.InspectContainer(name)
 }
 
-func (b *Build) getExportsContainer() (c *docker.Container, err error) {
-	name := b.exportsContainerName()
+func (b *Build) getExportsContainerWithBinds(name string, binds []string) (c *docker.Container, err error) {
 
 	config := &docker.Config{
 		Image: RsyncImage,
 		Volumes: map[string]struct{}{
 			"/opt/rsync/bin": struct{}{},
 			ExportsPath:      struct{}{},
 		},
+		Cmd:        []string{"/opt/rsync/bin/rsync", "-a", "--delete-during", "/.rocker_exports_source/", "/.rocker_exports/"},
+		Entrypoint: []string{},
+	}
+
+	var hostConfig *docker.HostConfig
+
+	if len(binds) != 0 {
+		hostConfig = &docker.HostConfig{
+			Binds: binds,
+		}
 	}
 
 	log.Debugf("Make EXPORT container %s with options %# v", name, config)
 
-	containerID, err := b.client.EnsureContainer(name, config, "exports")
+	containerID, err := b.client.EnsureContainer(name, config, hostConfig, "exports")
 	if err != nil {
 		return nil, err
 	}
@@ -251,6 +271,36 @@ func (b *Build) getExportsContainer() (c *docker.Container, err error) {
 	return b.client.InspectContainer(containerID)
 }
 
+func (b *Build) getExportsContainerAndSync(currentName, previousName string) (c *docker.Container, err error) {
+	//If it the first `EXPORT` in the file
+	//we don't need to sync data from previous container
+	if previousName == "" {
+		return b.getExportsContainer(currentName)
+	}
+
+	prevContainer, err := b.getExportsContainer(previousName)
+	if err != nil {
+		return nil, err
+	}
+
+	binds := mountsToBinds(prevContainer.Mounts, "_source")
+
+	currContainer, err := b.getExportsContainerWithBinds(currentName, binds)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Infof("| Running in %s: %s", currentName, strings.Join(currContainer.Config.Cmd, " "))
+	if err = b.client.RunContainer(currContainer.ID, false); err != nil {
+		return nil, err
+	}
+	return currContainer, nil
+}
+
+func (b *Build) getExportsContainer(name string) (c *docker.Container, err error) {
+	return b.getExportsContainerWithBinds(name, []string{})
+}
+
 // lookupImage looks up for the image by name and returns *docker.Image object (result of the inspect)
 // `Pull` config option defines whether we want to update the latest version of the image from the remote registry
 // See build.Config struct for more details about other build config options.

src/build/build_test.go

@@ -339,8 +339,8 @@ func (m *MockClient) EnsureImage(imageName string) error {
 	return args.Error(0)
 }
 
-func (m *MockClient) EnsureContainer(containerName string, config *docker.Config, purpose string) (containerID string, err error) {
-	args := m.Called(containerName, config, purpose)
+func (m *MockClient) EnsureContainer(containerName string, config *docker.Config, hostConfig *docker.HostConfig, purpose string) (containerID string, err error) {
+	args := m.Called(containerName, config, hostConfig, purpose)
 	return args.String(0), args.Error(1)
 }
 

src/build/client.go

@@ -56,7 +56,7 @@ type Client interface {
 	CommitContainer(state State, message string) (img *docker.Image, err error)
 	RemoveContainer(containerID string) error
 	UploadToContainer(containerID string, stream io.Reader, path string) error
-	EnsureContainer(containerName string, config *docker.Config, purpose string) (containerID string, err error)
+	EnsureContainer(containerName string, config *docker.Config, hostConfig *docker.HostConfig, purpose string) (containerID string, err error)
 	InspectContainer(containerName string) (*docker.Container, error)
 	ResolveHostPath(path string) (resultPath string, err error)
 }
@@ -576,7 +576,7 @@ func (c *DockerClient) EnsureImage(imageName string) (err error) {
 
 // EnsureContainer checks if container with specified name exists
 // and creates it otherwise
-func (c *DockerClient) EnsureContainer(containerName string, config *docker.Config, purpose string) (containerID string, err error) {
+func (c *DockerClient) EnsureContainer(containerName string, config *docker.Config, hostConfig *docker.HostConfig, purpose string) (containerID string, err error) {
 
 	// Check if container exists
 	container, err := c.client.InspectContainer(containerName)
@@ -597,8 +597,9 @@ func (c *DockerClient) EnsureContainer(containerName string, config *docker.Conf
 	c.log.Infof("| Create container: %s for %s", containerName, purpose)
 
 	opts := docker.CreateContainerOptions{
-		Name:   containerName,
-		Config: config,
+		Name:       containerName,
+		Config:     config,
+		HostConfig: hostConfig,
 	}
 
 	c.log.Debugf("Create container options %# v", opts)

src/build/commands.go

@@ -1141,7 +1141,7 @@ func (c *CommandMount) Execute(b *Build) (s State, err error) {
 			}
 
 			s.NoCache.HostConfig.Binds = append(s.NoCache.HostConfig.Binds,
-				mountsToBinds(c.Mounts)...)
+				mountsToBinds(c.Mounts, "")...)
 
 			commitIds = append(commitIds, strings.TrimLeft(c.Name, "/")+":"+arg)
 		}
@@ -1192,42 +1192,48 @@ func (c *CommandExport) Execute(b *Build) (s State, err error) {
 	// EXPORT /my/dir /stuff/ --> /EXPORT_VOLUME/stuff/my_dir
 	// EXPORT /my/dir/* / --> /EXPORT_VOLUME/stuff/my_dir
 
-	exportsContainer, err := b.getExportsContainer()
-	if err != nil {
-		return s, err
-	}
+	s.Commit("EXPORT %q to %s, prev_export_container_salt: %s", src, dest, b.prevExportContainerID)
 
 	// build the command
 	cmdDestPath, err := util.ResolvePath(ExportsPath, dest)
 	if err != nil {
 		return s, fmt.Errorf("Invalid EXPORT destination: %s", dest)
 	}
 
-	s.Commit("EXPORT %q to %.12s:%s", src, exportsContainer.ID, dest)
-
-	s, hit, err := b.probeCache(s)
+	s, hit, err := b.probeCacheAndPreserveCommits(s)
 	if err != nil {
 		return s, err
 	}
 	if hit {
-		b.exports = append(b.exports, s.ExportsID)
+		b.prevExportContainerID = s.ExportsID
+		b.currentExportContainerName = exportsContainerName(s.ParentID, s.GetCommits())
+		log.Infof("| Export container: %s", b.currentExportContainerName)
+		log.Debugf("===EXPORT CONTAINER NAME: %s ('%s', '%s')", b.currentExportContainerName, s.ParentID, s.GetCommits())
+		s.CleanCommits()
 		return s, nil
 	}
 
+	prevExportContainerName := b.currentExportContainerName
+	b.currentExportContainerName = exportsContainerName(s.ImageID, s.GetCommits())
+
+	exportsContainer, err := b.getExportsContainerAndSync(b.currentExportContainerName, prevExportContainerName)
+	if err != nil {
+		return s, err
+	}
+
 	// Remember original stuff so we can restore it when we finished
 	var exportsID string
 	origState := s
 
 	defer func() {
 		s = origState
 		s.ExportsID = exportsID
-		b.exports = append(b.exports, exportsID)
+		b.prevExportContainerID = exportsID
 	}()
 
 	// Append exports container as a volume
 	s.NoCache.HostConfig.Binds = append(s.NoCache.HostConfig.Binds,
-		mountsToBinds(exportsContainer.Mounts)...)
-
+		mountsToBinds(exportsContainer.Mounts, "")...)
 	cmd := []string{"/opt/rsync/bin/rsync", "-a", "--delete-during"}
 
 	if b.cfg.Verbose {
@@ -1277,7 +1283,7 @@ func (c *CommandImport) Execute(b *Build) (s State, err error) {
 	if len(args) == 0 {
 		return s, fmt.Errorf("IMPORT requires at least one argument")
 	}
-	if len(b.exports) == 0 {
+	if b.prevExportContainerID == "" {
 		return s, fmt.Errorf("You have to EXPORT something first in order to IMPORT")
 	}
 
@@ -1286,13 +1292,16 @@ func (c *CommandImport) Execute(b *Build) (s State, err error) {
 	// 			 because it was built earlier with the same prerequisites, but the actual
 	// 			 data in the exports container may be from the latest EXPORT of different
 	// 			 build. So we need to prefix ~/.rocker_exports dir with some id somehow.
+	if b.currentExportContainerName == "" {
+		return s, fmt.Errorf("You have to EXPORT something first to do IMPORT")
+	}
 
-	exportsContainer, err := b.getExportsContainer()
+	exportsContainer, err := b.getExportsContainer(b.currentExportContainerName)
 	if err != nil {
 		return s, err
 	}
 
-	log.Infof("| Import from %s (%.12s)", b.exportsContainerName(), exportsContainer.ID)
+	log.Infof("| Import from %s (%.12s)", b.currentExportContainerName, exportsContainer.ID)
 
 	// If only one argument was given to IMPORT, use the same path for destination
 	// IMPORT /my/dir/file.tar --> ADD ./EXPORT_VOLUME/my/dir/file.tar /my/dir/file.tar
@@ -1310,8 +1319,7 @@ func (c *CommandImport) Execute(b *Build) (s State, err error) {
 		src = append(src, argResolved)
 	}
 
-	sort.Strings(b.exports)
-	s.Commit("IMPORT %q : %q %s", b.exports, src, dest)
+	s.Commit("IMPORT %q : %q %s", b.prevExportContainerID, src, dest)
 
 	// Check cache
 	s, hit, err := b.probeCache(s)
@@ -1346,7 +1354,7 @@ func (c *CommandImport) Execute(b *Build) (s State, err error) {
 
 	// Append exports container as a volume
 	s.NoCache.HostConfig.Binds = append(s.NoCache.HostConfig.Binds,
-		mountsToBinds(exportsContainer.Mounts)...)
+		mountsToBinds(exportsContainer.Mounts, "")...)
 
 	if importID, err = b.client.CreateContainer(s); err != nil {
 		return s, err

src/build/commands_test.go

@@ -669,7 +669,7 @@ func TestCommandMount_VolumeContainer(t *testing.T) {
 
 	containerName := b.mountsContainerName("/cache")
 
-	c.On("EnsureContainer", containerName, mock.AnythingOfType("*docker.Config"), "/cache").Return("123", nil).Run(func(args mock.Arguments) {
+	c.On("EnsureContainer", containerName, mock.AnythingOfType("*docker.Config"), mock.AnythingOfType("*docker.HostConfig"), "/cache").Return("123", nil).Run(func(args mock.Arguments) {
 		arg := args.Get(1).(*docker.Config)
 		assert.Equal(t, MountVolumeImage, arg.Image)
 		expectedVolumes := map[string]struct{}{

src/build/util.go

@@ -31,12 +31,6 @@ func (b *Build) mountsContainerName(path string) string {
 	return fmt.Sprintf("rocker_mount_%.6x", md5.Sum([]byte(mountID)))
 }
 
-// exportsContainerName return the name of volume container that will be used for EXPORTs
-func (b *Build) exportsContainerName() string {
-	mountID := b.getIdentifier()
-	return fmt.Sprintf("rocker_exports_%.6x", md5.Sum([]byte(mountID)))
-}
-
 // getIdentifier returns the sequence that is unique to the current Rockerfile
 func (b *Build) getIdentifier() string {
 	if b.cfg.ID != "" {
@@ -46,21 +40,28 @@ func (b *Build) getIdentifier() string {
 }
 
 // mountsToBinds turns the list of mounts to the list of binds
-func mountsToBinds(mounts []docker.Mount) []string {
+func mountsToBinds(mounts []docker.Mount, prefix string) []string {
 	result := make([]string, len(mounts))
 	for i, m := range mounts {
 		// TODO: Mode?
-		result[i] = mountToBind(m, m.RW)
+		result[i] = mountToBind(m, m.RW, prefix)
 	}
 	return result
 }
 
+// exportsContainerName return the name of volume container that will be used for EXPORTs
+func exportsContainerName(imageID string, commits string) string {
+	mountID := imageID + commits
+	name := fmt.Sprintf("rocker_exports_%.12x", md5.Sum([]byte(mountID)))
+	return name
+}
+
 // mountToBind turns docker.Mount into a bind string
-func mountToBind(m docker.Mount, rw bool) string {
+func mountToBind(m docker.Mount, rw bool, prefix string) string {
 	if rw {
-		return m.Source + ":" + m.Destination + ":rw"
+		return m.Source + ":" + m.Destination + prefix + ":rw"
 	}
-	return m.Source + ":" + m.Destination + ":ro"
+	return m.Source + ":" + m.Destination + prefix + ":ro"
 }
 
 // readerVoidCloser is a hack of the improved go-dockerclient's hijacking behavior