setns: replace env with netlink for bootstrap data

replace passing of pid and console path via environment variable with passing
them with netlink message via an established pipe.

this change requires us to set _LIBCONTAINER_INITTYPE and
_LIBCONTAINER_INITPIPE as the env environment of the bootstrap process as we
only send the bootstrap data for setns process right now. When init and setns
bootstrap process are unified (i.e., init use nsexec instead of Go to clone new
process), we can remove _LIBCONTAINER_INITTYPE.

Note:
- we read nlmsghdr first before reading the content so we can get the total
  length of the payload and allocate buffer properly instead of allocating
  one large buffer.

- check read bytes vs the wanted number. It's an error if we failed to read
  the desired number of bytes from the pipe into the buffer.

Signed-off-by: Daniel, Dao Quang Minh <dqminh89@gmail.com>

libcontainer/container_linux.go

@@ -3,8 +3,10 @@
 package libcontainer
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -19,6 +21,7 @@ import (
 	"github.com/opencontainers/runc/libcontainer/cgroups"
 	"github.com/opencontainers/runc/libcontainer/configs"
 	"github.com/opencontainers/runc/libcontainer/criurpc"
+	"github.com/vishvananda/netlink/nl"
 )
 
 const stdioFdCount = 3
@@ -274,21 +277,22 @@ func (c *linuxContainer) newInitProcess(p *Process, cmd *exec.Cmd, parentPipe, c
 }
 
 func (c *linuxContainer) newSetnsProcess(p *Process, cmd *exec.Cmd, parentPipe, childPipe *os.File) (*setnsProcess, error) {
-	cmd.Env = append(cmd.Env,
-		fmt.Sprintf("_LIBCONTAINER_INITPID=%d", c.initProcess.pid()),
-		"_LIBCONTAINER_INITTYPE=setns",
-	)
-	if p.consolePath != "" {
-		cmd.Env = append(cmd.Env, "_LIBCONTAINER_CONSOLE_PATH="+p.consolePath)
+	cmd.Env = append(cmd.Env, "_LIBCONTAINER_INITTYPE=setns")
+	// for setns process, we dont have to set cloneflags as the process namespaces
+	// will only be set via setns syscall
+	data, err := c.bootstrapData(0, c.initProcess.pid(), p.consolePath)
+	if err != nil {
+		return nil, err
 	}
 	// TODO: set on container for process management
 	return &setnsProcess{
-		cmd:         cmd,
-		cgroupPaths: c.cgroupManager.GetPaths(),
-		childPipe:   childPipe,
-		parentPipe:  parentPipe,
-		config:      c.newInitConfig(p),
-		process:     p,
+		cmd:           cmd,
+		cgroupPaths:   c.cgroupManager.GetPaths(),
+		childPipe:     childPipe,
+		parentPipe:    parentPipe,
+		config:        c.newInitConfig(p),
+		process:       p,
+		bootstrapData: data,
 	}, nil
 }
 
@@ -1021,3 +1025,25 @@ func (c *linuxContainer) currentState() (*State, error) {
 	}
 	return state, nil
 }
+
+// bootstrapData encodes the necessary data in netlink binary format as a io.Reader.
+// Consumer can write the data to a bootstrap program such as one that uses
+// nsenter package to bootstrap the container's init process correctly, i.e. with
+// correct namespaces, uid/gid mapping etc.
+func (c *linuxContainer) bootstrapData(cloneFlags uintptr, pid int, consolePath string) (io.Reader, error) {
+	// create the netlink message
+	r := nl.NewNetlinkRequest(int(InitMsg), 0)
+	// write pid
+	r.AddData(&Int32msg{
+		Type:  PidAttr,
+		Value: uint32(pid),
+	})
+	// write console path
+	if consolePath != "" {
+		r.AddData(&Bytemsg{
+			Type:  ConsolePathAttr,
+			Value: []byte(consolePath),
+		})
+	}
+	return bytes.NewReader(r.Serialize()), nil
+}

libcontainer/message_linux.go

@@ -0,0 +1,59 @@
+// +build linux
+package libcontainer
+
+import (
+	"syscall"
+
+	"github.com/vishvananda/netlink/nl"
+)
+
+// list of known message types we want to send to bootstrap program
+// The number is randomly chosen to not conflict with known netlink types
+const (
+	InitMsg         uint16 = 62000
+	PidAttr         uint16 = 27281
+	ConsolePathAttr uint16 = 27282
+)
+
+type Int32msg struct {
+	Type  uint16
+	Value uint32
+}
+
+// int32msg has the following representation
+// | nlattr len | nlattr type |
+// | uint32 value             |
+func (msg *Int32msg) Serialize() []byte {
+	buf := make([]byte, msg.Len())
+	native := nl.NativeEndian()
+	native.PutUint16(buf[0:2], uint16(msg.Len()))
+	native.PutUint16(buf[2:4], msg.Type)
+	native.PutUint32(buf[4:8], msg.Value)
+	return buf
+}
+
+func (msg *Int32msg) Len() int {
+	return syscall.NLA_HDRLEN + 4
+}
+
+// bytemsg has the following representation
+// | nlattr len | nlattr type |
+// | value              | pad |
+type Bytemsg struct {
+	Type  uint16
+	Value []byte
+}
+
+func (msg *Bytemsg) Serialize() []byte {
+	l := msg.Len()
+	buf := make([]byte, (l+syscall.NLA_ALIGNTO-1) & ^(syscall.NLA_ALIGNTO-1))
+	native := nl.NativeEndian()
+	native.PutUint16(buf[0:2], uint16(l))
+	native.PutUint16(buf[2:4], msg.Type)
+	copy(buf[4:], msg.Value)
+	return buf
+}
+
+func (msg *Bytemsg) Len() int {
+	return syscall.NLA_HDRLEN + len(msg.Value) + 1 // null-terminated
+}

libcontainer/nsenter/nsenter_test.go

@@ -1,12 +1,17 @@
 package nsenter
 
 import (
+	"bytes"
 	"encoding/json"
-	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"strings"
+	"syscall"
 	"testing"
+
+	"github.com/opencontainers/runc/libcontainer"
+	"github.com/vishvananda/netlink/nl"
 )
 
 type pid struct {
@@ -15,24 +20,30 @@ type pid struct {
 
 func TestNsenterAlivePid(t *testing.T) {
 	args := []string{"nsenter-exec"}
-	r, w, err := os.Pipe()
+	parent, child, err := newPipe()
 	if err != nil {
 		t.Fatalf("failed to create pipe %v", err)
 	}
 
 	cmd := &exec.Cmd{
 		Path:       os.Args[0],
 		Args:       args,
-		ExtraFiles: []*os.File{w},
-		Env:        []string{fmt.Sprintf("_LIBCONTAINER_INITPID=%d", os.Getpid()), "_LIBCONTAINER_INITPIPE=3"},
+		ExtraFiles: []*os.File{child},
+		Env:        []string{"_LIBCONTAINER_INITTYPE=setns", "_LIBCONTAINER_INITPIPE=3"},
 	}
 
 	if err := cmd.Start(); err != nil {
 		t.Fatalf("nsenter failed to start %v", err)
 	}
-	w.Close()
-
-	decoder := json.NewDecoder(r)
+	r := nl.NewNetlinkRequest(int(libcontainer.InitMsg), 0)
+	r.AddData(&libcontainer.Int32msg{
+		Type:  libcontainer.PidAttr,
+		Value: uint32(os.Getpid()),
+	})
+	if _, err := io.Copy(parent, bytes.NewReader(r.Serialize())); err != nil {
+		t.Fatal(err)
+	}
+	decoder := json.NewDecoder(parent)
 	var pid *pid
 
 	if err := decoder.Decode(&pid); err != nil {
@@ -51,34 +62,67 @@ func TestNsenterAlivePid(t *testing.T) {
 
 func TestNsenterInvalidPid(t *testing.T) {
 	args := []string{"nsenter-exec"}
+	parent, child, err := newPipe()
+	if err != nil {
+		t.Fatalf("failed to create pipe %v", err)
+	}
 
 	cmd := &exec.Cmd{
-		Path: os.Args[0],
-		Args: args,
-		Env:  []string{"_LIBCONTAINER_INITPID=-1"},
+		Path:       os.Args[0],
+		Args:       args,
+		ExtraFiles: []*os.File{child},
+		Env:        []string{"_LIBCONTAINER_INITTYPE=setns", "_LIBCONTAINER_INITPIPE=3"},
+	}
+
+	if err := cmd.Start(); err != nil {
+		t.Fatal("nsenter exits with a zero exit status")
+	}
+	r := nl.NewNetlinkRequest(int(libcontainer.InitMsg), 0)
+	r.AddData(&libcontainer.Int32msg{
+		Type:  libcontainer.PidAttr,
+		Value: 0,
+	})
+	if _, err := io.Copy(parent, bytes.NewReader(r.Serialize())); err != nil {
+		t.Fatal(err)
 	}
 
-	err := cmd.Run()
-	if err == nil {
+	if err := cmd.Wait(); err == nil {
 		t.Fatal("nsenter exits with a zero exit status")
 	}
 }
 
 func TestNsenterDeadPid(t *testing.T) {
-	dead_cmd := exec.Command("true")
-	if err := dead_cmd.Run(); err != nil {
+	deadCmd := exec.Command("true")
+	if err := deadCmd.Run(); err != nil {
 		t.Fatal(err)
 	}
 	args := []string{"nsenter-exec"}
+	parent, child, err := newPipe()
+	if err != nil {
+		t.Fatalf("failed to create pipe %v", err)
+	}
 
 	cmd := &exec.Cmd{
-		Path: os.Args[0],
-		Args: args,
-		Env:  []string{fmt.Sprintf("_LIBCONTAINER_INITPID=%d", dead_cmd.Process.Pid)},
+		Path:       os.Args[0],
+		Args:       args,
+		ExtraFiles: []*os.File{child},
+		Env:        []string{"_LIBCONTAINER_INITTYPE=setns", "_LIBCONTAINER_INITPIPE=3"},
 	}
 
-	err := cmd.Run()
-	if err == nil {
+	if err := cmd.Start(); err != nil {
+		t.Fatal("nsenter exits with a zero exit status")
+	}
+
+	r := nl.NewNetlinkRequest(int(libcontainer.InitMsg), 0)
+	r.AddData(&libcontainer.Int32msg{
+		Type:  libcontainer.PidAttr,
+		Value: uint32(deadCmd.Process.Pid),
+	})
+	if _, err := io.Copy(parent, bytes.NewReader(r.Serialize())); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := cmd.Wait(); err == nil {
 		t.Fatal("nsenter exits with a zero exit status")
 	}
 }
@@ -89,3 +133,11 @@ func init() {
 	}
 	return
 }
+
+func newPipe() (parent *os.File, child *os.File, err error) {
+	fds, err := syscall.Socketpair(syscall.AF_LOCAL, syscall.SOCK_STREAM|syscall.SOCK_CLOEXEC, 0)
+	if err != nil {
+		return nil, nil, err
+	}
+	return os.NewFile(uintptr(fds[1]), "parent"), os.NewFile(uintptr(fds[0]), "child"), nil
+}