Fix exec with tty

go.mod

@@ -5,9 +5,9 @@ go 1.19
 require (
 	github.com/blang/semver/v4 v4.0.0
 	github.com/compose-spec/compose-go v1.8.2
-	github.com/containerd/console v1.0.3
 	github.com/containerd/go-cni v1.1.7
 	github.com/containernetworking/plugins v1.2.0
+	github.com/creack/pty v1.1.18
 	github.com/emicklei/go-restful/v3 v3.10.1
 	github.com/google/go-cmp v0.5.9
 	github.com/itchyny/gojq v0.12.11

go.sum

@@ -56,8 +56,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/compose-spec/compose-go v1.8.2 h1:sUQvDxnPgpcOyoxC/lz7mFTrTlHeZ6LWyuASYetkOqw=
 github.com/compose-spec/compose-go v1.8.2/go.mod h1:Tb5Ae2PsYN3GTqYqzl2IRbTPiJtPZZjMw8UKUvmehFk=
-github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw=
-github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
 github.com/containerd/go-cni v1.1.7 h1:1yKpVCQAXI21BJIy8q7Nyk4CWpIgUno6ib7JIDca7D4=
 github.com/containerd/go-cni v1.1.7/go.mod h1:Ve4Q0RB2Bw78D90OL0YVyDjqdTL7FKh9W+UPbhWiZXA=
 github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ=
@@ -67,6 +65,8 @@ github.com/containernetworking/plugins v1.2.0/go.mod h1:/VjX4uHecW5vVimFa1wkG4s+
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
+github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

pkg/kwok/server/debugging_exec.go

@@ -63,13 +63,18 @@ func (s *Server) ExecInContainer(ctx context.Context, podName, podNamespace stri
 	defer cancel()
 
 	if tty {
-		return s.execInContainerWithTTY(ctx, cmd, in, out, errOut, resize)
+		return s.execInContainerWithTTY(ctx, cmd, in, out, resize)
 	}
 
 	return s.execInContainer(ctx, cmd, in, out, errOut)
 }
 
 func (s *Server) execInContainer(ctx context.Context, cmd []string, in io.Reader, out, errOut io.WriteCloser) error {
+	// Set the pipe stdin.
+	if in != nil {
+		ctx = exec.WithPipeStdin(ctx, true)
+	}
+
 	// Set the stream as the stdin/stdout/stderr.
 	ctx = exec.WithIOStreams(ctx, exec.IOStreams{
 		In:     in,

pkg/kwok/server/debugging_exec_other.go

@@ -21,38 +21,25 @@ package server
 import (
 	"context"
 	"io"
-	"os"
 
-	"github.com/containerd/console"
+	cpty "github.com/creack/pty"
 	clientremotecommand "k8s.io/client-go/tools/remotecommand"
 
 	"sigs.k8s.io/kwok/pkg/log"
 	"sigs.k8s.io/kwok/pkg/utils/exec"
 )
 
-func (s *Server) execInContainerWithTTY(ctx context.Context, cmd []string, in io.Reader, out, errOut io.WriteCloser, resize <-chan clientremotecommand.TerminalSize) error {
+func (s *Server) execInContainerWithTTY(ctx context.Context, cmd []string, in io.Reader, out io.WriteCloser, resize <-chan clientremotecommand.TerminalSize) error {
 	logger := log.FromContext(ctx)
 
 	// Create a pty.
-	pty, slavePath, err := console.NewPty()
+	pty, tty, err := cpty.Open()
 	if err != nil {
 		return err
 	}
 	defer func() {
 		_ = pty.Close()
-	}()
-	err = pty.SetRaw()
-	if err != nil {
-		return err
-	}
-
-	// Open the slave side of the pty.
-	slave, err := os.OpenFile(slavePath, os.O_RDWR, 0)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		_ = slave.Close()
+		_ = tty.Close()
 	}()
 
 	// Create a two way tunnel for pty and stream.
@@ -77,10 +64,11 @@ func (s *Server) execInContainerWithTTY(ctx context.Context, cmd []string, in io
 	if resize != nil {
 		go func() {
 			for size := range resize {
-				if err := pty.Resize(console.WinSize{
-					Width:  size.Width,
-					Height: size.Height,
-				}); err != nil {
+				err := cpty.Setsize(pty, &cpty.Winsize{
+					Rows: size.Height,
+					Cols: size.Width,
+				})
+				if err != nil {
 					logger.Error("failed to resize pty", err)
 				}
 			}
@@ -89,9 +77,9 @@ func (s *Server) execInContainerWithTTY(ctx context.Context, cmd []string, in io
 
 	// Set the stream as the stdin/stdout/stderr.
 	ctx = exec.WithIOStreams(ctx, exec.IOStreams{
-		In:     slave,
-		Out:    slave,
-		ErrOut: slave,
+		In:     tty,
+		Out:    tty,
+		ErrOut: tty,
 	})
 
 	// Execute the command.

pkg/kwok/server/debugging_exec_windows.go

@@ -23,8 +23,12 @@ import (
 	"io"
 
 	clientremotecommand "k8s.io/client-go/tools/remotecommand"
+
+	"sigs.k8s.io/kwok/pkg/log"
 )
 
-func (s *Server) execInContainerWithTTY(ctx context.Context, cmd []string, in io.Reader, out, errOut io.WriteCloser, resize <-chan clientremotecommand.TerminalSize) error {
-	return s.execInContainer(ctx, cmd, in, out, errOut)
+func (s *Server) execInContainerWithTTY(ctx context.Context, cmd []string, in io.Reader, out io.WriteCloser, resize <-chan clientremotecommand.TerminalSize) error {
+	logger := log.FromContext(ctx)
+	logger.Warn("execInContainerWithTTY is not supported on windows, fallback to execInContainer")
+	return s.execInContainer(ctx, cmd, in, out, out)
 }

pkg/utils/exec/exec.go

@@ -44,6 +44,15 @@ type execOptions struct {
 	Env []string
 	// IOStreams contains the standard streams.
 	IOStreams
+	// PipeStdin is true if the command's stdin should be piped.
+	PipeStdin bool
+}
+
+// WithPipeStdin returns a context with the given pipeStdin option.
+func WithPipeStdin(ctx context.Context, pipeStdin bool) context.Context {
+	ctx, opt := withExecOptions(ctx)
+	opt.PipeStdin = pipeStdin
+	return ctx
 }
 
 // WithEnv returns a context with the given environment variables.
@@ -130,20 +139,39 @@ func Exec(ctx context.Context, name string, arg ...string) error {
 		cmd.Env = append(cmd.Env, os.Environ()...)
 	}
 	cmd.Dir = opt.Dir
-	cmd.Stdin = opt.In
+	if opt.In != nil {
+		if opt.PipeStdin {
+			inPipe, err := cmd.StdinPipe()
+			if err != nil {
+				return err
+			}
+			go func() {
+				_, _ = io.Copy(inPipe, opt.In)
+			}()
+		} else {
+			cmd.Stdin = opt.In
+		}
+	}
+
 	cmd.Stdout = opt.Out
 	cmd.Stderr = opt.ErrOut
 
 	if cmd.Stderr == nil {
 		buf := bytes.NewBuffer(nil)
 		cmd.Stderr = buf
 	}
-	err := cmd.Run()
+
+	err := cmd.Start()
+	if err != nil {
+		return fmt.Errorf("cmd start: %s %s: %w", name, strings.Join(arg, " "), err)
+	}
+
+	err = cmd.Wait()
 	if err != nil {
 		if buf, ok := cmd.Stderr.(*bytes.Buffer); ok {
-			return fmt.Errorf("%s %s: %w\n%s", name, strings.Join(arg, " "), err, buf.String())
+			return fmt.Errorf("cmd wait: %s %s: %w\n%s", name, strings.Join(arg, " "), err, buf.String())
 		}
-		return fmt.Errorf("%s %s: %w", name, strings.Join(arg, " "), err)
+		return fmt.Errorf("cmd wait: %s %s: %w", name, strings.Join(arg, " "), err)
 	}
 	return nil
 }

test/kwokctl/kwokctl_exec_test.sh

@@ -53,7 +53,7 @@ function test_exec() {
   local cmd="${4}"
   local want="${5}"
   local result
-  result=$(kwokctl --name "${name}" kubectl -n "${namespace}" exec "${target}" -- "${cmd}")
+  result=$(kwokctl --name "${name}" kubectl -n "${namespace}" exec -i "${target}" -- "${cmd}")
   if [[ $? -ne 0 ]]; then
       echo "Error: exec failed"
       return 1