Merge pull request #121 from thaJeztah/18.09_backport_containerd_v1.2.1

[18.09 backport] Update containerd to v1.2.1-rc.0

daemon/graphdriver/lcow/lcow.go

@@ -71,20 +71,33 @@ import (
 	"time"
 
 	"github.com/Microsoft/hcsshim"
+	"github.com/Microsoft/hcsshim/ext4/tar2ext4"
 	"github.com/Microsoft/opengcs/client"
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/ioutils"
+	"github.com/docker/docker/pkg/reexec"
 	"github.com/docker/docker/pkg/system"
 	"github.com/sirupsen/logrus"
 )
 
+// noreexec controls reexec functionality. Off by default, on for debugging purposes.
+var noreexec = false
+
 // init registers this driver to the register. It gets initialised by the
 // function passed in the second parameter, implemented in this file.
 func init() {
 	graphdriver.Register("lcow", InitDriver)
+	// DOCKER_LCOW_NOREEXEC allows for inline processing which makes
+	// debugging issues in the re-exec codepath significantly easier.
+	if os.Getenv("DOCKER_LCOW_NOREEXEC") != "" {
+		logrus.Warnf("LCOW Graphdriver is set to not re-exec. This is intended for debugging purposes only.")
+		noreexec = true
+	} else {
+		reexec.Register("docker-lcow-tar2ext4", tar2ext4Reexec)
+	}
 }
 
 const (
@@ -846,32 +859,72 @@ func (d *Driver) Diff(id, parent string) (io.ReadCloser, error) {
 func (d *Driver) ApplyDiff(id, parent string, diff io.Reader) (int64, error) {
 	logrus.Debugf("lcowdriver: applydiff: id %s", id)
 
-	svm, err := d.startServiceVMIfNotRunning(id, nil, fmt.Sprintf("applydiff %s", id))
+	// Log failures here as it's undiagnosable sometimes, due to a possible panic.
+	// See https://github.com/moby/moby/issues/37955 for more information.
+
+	dest := filepath.Join(d.dataRoot, id, layerFilename)
+	if !noreexec {
+		cmd := reexec.Command([]string{"docker-lcow-tar2ext4", dest}...)
+		stdout := bytes.NewBuffer(nil)
+		stderr := bytes.NewBuffer(nil)
+		cmd.Stdin = diff
+		cmd.Stdout = stdout
+		cmd.Stderr = stderr
+
+		if err := cmd.Start(); err != nil {
+			logrus.Warnf("lcowdriver: applydiff: id %s failed to start re-exec: %s", id, err)
+			return 0, err
+		}
+
+		if err := cmd.Wait(); err != nil {
+			logrus.Warnf("lcowdriver: applydiff: id %s failed %s", id, err)
+			return 0, fmt.Errorf("re-exec error: %v: stderr: %s", err, stderr)
+		}
+		return strconv.ParseInt(stdout.String(), 10, 64)
+	}
+	// The inline case
+	size, err := tar2ext4Actual(dest, diff)
 	if err != nil {
-		return 0, err
+		logrus.Warnf("lcowdriver: applydiff: id %s failed %s", id, err)
 	}
-	defer d.terminateServiceVM(id, fmt.Sprintf("applydiff %s", id), false)
+	return size, err
+}
 
-	logrus.Debugf("lcowdriver: applydiff: waiting for svm to finish booting")
-	err = svm.getStartError()
+// tar2ext4Reexec is the re-exec entry point for writing a layer from a tar file
+func tar2ext4Reexec() {
+	size, err := tar2ext4Actual(os.Args[1], os.Stdin)
 	if err != nil {
-		return 0, fmt.Errorf("lcowdriver: applydiff: svm failed to boot: %s", err)
+		fmt.Fprint(os.Stderr, err)
+		os.Exit(1)
 	}
+	fmt.Fprint(os.Stdout, size)
+}
 
-	// TODO @jhowardmsft - the retries are temporary to overcome platform reliability issues.
-	// Obviously this will be removed as platform bugs are fixed.
-	retries := 0
-	for {
-		retries++
-		size, err := svm.config.TarToVhd(filepath.Join(d.dataRoot, id, layerFilename), diff)
-		if err != nil {
-			if retries <= 10 {
-				continue
-			}
-			return 0, err
-		}
-		return size, err
+// tar2ext4Actual is the implementation of tar2ext to write a layer from a tar file.
+// It can be called through re-exec (default), or inline for debugging.
+func tar2ext4Actual(dest string, diff io.Reader) (int64, error) {
+	// maxDiskSize is not relating to the sandbox size - this is the
+	// maximum possible size a layer VHD generated can be from an EXT4
+	// layout perspective.
+	const maxDiskSize = 128 * 1024 * 1024 * 1024 // 128GB
+	out, err := os.Create(dest)
+	if err != nil {
+		return 0, err
+	}
+	defer out.Close()
+	if err := tar2ext4.Convert(
+		diff,
+		out,
+		tar2ext4.AppendVhdFooter,
+		tar2ext4.ConvertWhiteout,
+		tar2ext4.MaximumDiskSize(maxDiskSize)); err != nil {
+		return 0, err
+	}
+	fi, err := os.Stat(dest)
+	if err != nil {
+		return 0, err
 	}
+	return fi.Size(), nil
 }
 
 // Changes produces a list of changes between the specified layer

hack/dockerfile/install/containerd.installer

@@ -4,7 +4,7 @@
 # containerd is also pinned in vendor.conf. When updating the binary
 # version you may also need to update the vendor version to pick up bug
 # fixes or new APIs.
-CONTAINERD_COMMIT=468a545b9edcd5932818eb9de8e72413e616e86e # v1.1.2
+CONTAINERD_COMMIT=de1f167ab96338a9f5c2b17347abf84bdf1dd411 # v1.2.1-rc.0
 
 install_containerd() {
 	echo "Install containerd version $CONTAINERD_COMMIT"

hack/dockerfile/install/runc.installer

@@ -1,13 +1,21 @@
 #!/bin/sh
 
 # When updating RUNC_COMMIT, also update runc in vendor.conf accordingly
-RUNC_COMMIT=69663f0bd4b60df09991c08812a60108003fa340
+# The version of runc should match the version that is used by the containerd
+# version that is used. If you need to update runc, open a pull request in
+# the containerd project first, and update both after that is merged.
+RUNC_COMMIT=10d38b660a77168360df3522881e2dc2be5056bd
 
 install_runc() {
+	# If using RHEL7 kernels (3.10.0 el7), disable kmem accounting/limiting
+	if uname -r | grep -q '^3\.10\.0.*\.el7\.'; then
+		: ${RUNC_NOKMEM='nokmem'}
+	fi
+
 	# Do not build with ambient capabilities support
-	RUNC_BUILDTAGS="${RUNC_BUILDTAGS:-"seccomp apparmor selinux"}"
+	RUNC_BUILDTAGS="${RUNC_BUILDTAGS:-"seccomp apparmor selinux $RUNC_NOKMEM"}"
 
-	echo "Install runc version $RUNC_COMMIT"
+	echo "Install runc version $RUNC_COMMIT (build tags: $RUNC_BUILDTAGS)"
 	git clone https://github.com/opencontainers/runc.git "$GOPATH/src/github.com/opencontainers/runc"
 	cd "$GOPATH/src/github.com/opencontainers/runc"
 	git checkout -q "$RUNC_COMMIT"

libcontainerd/supervisor/remote_daemon.go

@@ -15,7 +15,7 @@ import (
 
 	"github.com/BurntSushi/toml"
 	"github.com/containerd/containerd"
-	"github.com/containerd/containerd/services/server"
+	"github.com/containerd/containerd/services/server/config"
 	"github.com/docker/docker/pkg/system"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -37,7 +37,7 @@ type pluginConfigs struct {
 
 type remote struct {
 	sync.RWMutex
-	server.Config
+	config.Config
 
 	daemonPid int
 	logger    *logrus.Entry
@@ -65,7 +65,7 @@ func Start(ctx context.Context, rootDir, stateDir string, opts ...DaemonOpt) (Da
 	r := &remote{
 		rootDir:  rootDir,
 		stateDir: stateDir,
-		Config: server.Config{
+		Config: config.Config{
 			Root:  filepath.Join(rootDir, "daemon"),
 			State: filepath.Join(stateDir, "daemon"),
 		},

vendor.conf

@@ -1,6 +1,6 @@
 # the following lines are in sorted order, FYI
 github.com/Azure/go-ansiterm d6e3b3328b783f23731bc4d058875b0371ff8109
-github.com/Microsoft/hcsshim v0.7.6
+github.com/Microsoft/hcsshim v0.7.12
 github.com/Microsoft/go-winio v0.4.11
 github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a
 github.com/go-check/check 4ed411733c5785b40214c70bce814c3a3a689609 https://github.com/cpuguy83/check.git
@@ -74,9 +74,13 @@ github.com/pborman/uuid v1.0
 
 google.golang.org/grpc v1.12.0
 
-# This does not need to match RUNC_COMMIT as it is used for helper packages but should be newer or equal
-github.com/opencontainers/runc 00dc70017d222b178a002ed30e9321b12647af2d
-github.com/opencontainers/runtime-spec eba862dc2470385a233c7507392675cbeadf7353 # v1.0.1-45-geba862d
+# The version of runc should match the version that is used by the containerd
+# version that is used. If you need to update runc, open a pull request in
+# the containerd project first, and update both after that is merged.
+# This commit does not need to match RUNC_COMMIT as it is used for helper
+# packages but should be newer or equal.
+github.com/opencontainers/runc 10d38b660a77168360df3522881e2dc2be5056bd
+github.com/opencontainers/runtime-spec 5684b8af48c1ac3b1451fa499724e30e3c20a294 # v1.0.1-49-g5684b8a
 github.com/opencontainers/image-spec v1.0.1
 github.com/seccomp/libseccomp-golang 32f571b70023028bd57d9288c20efbcb237f3ce0
 
@@ -114,12 +118,12 @@ github.com/googleapis/gax-go v2.0.0
 google.golang.org/genproto 694d95ba50e67b2e363f3483057db5d4910c18f9
 
 # containerd
-github.com/containerd/containerd 0c5f8f63c3368856c320ae8a1c125e703b73b51d # v1.2.0-rc.1
+github.com/containerd/containerd de1f167ab96338a9f5c2b17347abf84bdf1dd411 # v1.2.1-rc.0
 github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c
 github.com/containerd/continuity bd77b46c8352f74eb12c85bdc01f4b90f69d66b4
 github.com/containerd/cgroups 5e610833b72089b37d0e615de9a92dfc043757c2
 github.com/containerd/console c12b1e7919c14469339a5d38f2f8ed9b64a9de23
-github.com/containerd/cri 9f39e3289533fc228c5e5fcac0a6dbdd60c6047b # release/1.2 branch
+github.com/containerd/cri f913714917d2456d7e65a0be84962b1ce8acb487 # release/1.2 branch
 github.com/containerd/go-runc 5a6d9f37cfa36b15efba46dc7ea349fa9b7143c3
 github.com/containerd/typeurl a93fcdb778cd272c6e9b3028b2f42d813e785d40
 github.com/containerd/ttrpc 2a805f71863501300ae1976d29f0454ae003e85a