[kwokctl] Fix binary runtime on windows

.github/workflows/test.yaml

@@ -106,24 +106,32 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os:
-          - ubuntu-latest
-          - macos-latest
+        include:
+          # Windows
           # Linux containers are not supported on Windows Runner
           # https://github.com/orgs/community/discussions/25491#discussioncomment-3248089
-          # - windows-latest
-        kwokctl-runtime:
-          - binary
-          - docker
-          - podman
-          - kind
-          - kind-podman
-        exclude:
+          - os: windows-latest
+            kwokctl-runtime: binary
+
+          # MacOS
           - os: macos-latest
-            kwokctl-runtime: podman
+            kwokctl-runtime: binary
+          - os: macos-latest
+            kwokctl-runtime: docker
           - os: macos-latest
+            kwokctl-runtime: kind
+
+          # Linux
+          - os: ubuntu-latest
+            kwokctl-runtime: binary
+          - os: ubuntu-latest
+            kwokctl-runtime: docker
+          - os: ubuntu-latest
+            kwokctl-runtime: podman
+          - os: ubuntu-latest
+            kwokctl-runtime: kind
+          - os: ubuntu-latest
             kwokctl-runtime: kind-podman
-        include:
           # nerdctl is still very early and has many bugs and differences in docker behavior,
           # so we need to verify that it works on every release.
           - os: ubuntu-latest
@@ -158,22 +166,33 @@ jobs:
         # https://www.downloadkubernetes.com
       - name: Download Kubernetes Source Code
         uses: actions/checkout@v3
-        if: ${{ matrix.kwokctl-runtime == 'binary' && matrix.os == 'macos-latest' }}
+        if: ${{ matrix.kwokctl-runtime == 'binary' && ( matrix.os == 'macos-latest' || matrix.os == 'windows-latest' ) }}
         with:
           repository: kubernetes/kubernetes
           path: kubernetes
           ref: v1.27.3
-      - name: Build Kubernetes Binary
+      - name: Build Kubernetes Binary for Windows
+        if: ${{ matrix.kwokctl-runtime == 'binary' && matrix.os == 'windows-latest' }}
+        shell: bash
+        run: |
+          export GOBIN="$(go env GOPATH)/bin"
+          cd "${GITHUB_WORKSPACE}/kubernetes" && go install ./cmd/{kube-apiserver,kube-controller-manager,kube-scheduler}
+          mkdir -p ~/.kwok && cat > ~/.kwok/kwok.yaml << EOF
+          kind: KwokctlConfiguration
+          apiVersion: config.kwok.x-k8s.io/v1alpha1
+          options:
+            kubeBinaryPrefix: '${GOBIN//\\//}'
+          EOF
+      - name: Build Kubernetes Binary for MacOS
         if: ${{ matrix.kwokctl-runtime == 'binary' && matrix.os == 'macos-latest' }}
         shell: bash
         run: |
           cd "${GITHUB_WORKSPACE}/kubernetes" && make WHAT="cmd/kube-apiserver cmd/kube-controller-manager cmd/kube-scheduler"
-          mkdir -p ~/.kwok
-          cat > ~/.kwok/kwok.yaml << EOF
+          mkdir -p ~/.kwok && cat > ~/.kwok/kwok.yaml << EOF
           kind: KwokctlConfiguration
           apiVersion: config.kwok.x-k8s.io/v1alpha1
           options:
-            kubeBinaryPrefix: "${GITHUB_WORKSPACE}/kubernetes/_output/bin"
+            kubeBinaryPrefix: '${GITHUB_WORKSPACE}/kubernetes/_output/bin'
           EOF
 
         # TODO: workaround for https://github.com/actions/runner-images/issues/7753 (caused by https://bugs.launchpad.net/ubuntu/+source/libpod/+bug/2024394).
@@ -228,7 +247,7 @@ jobs:
           sudo systemctl daemon-reload
 
       - name: Make pki directory
-        if: ${{ matrix.kwokctl-runtime == 'binary' }}
+        if: ${{ matrix.kwokctl-runtime == 'binary' && matrix.os != 'windows-latest' }}
         shell: bash
         run: |
           sudo mkdir -p /var/run/kubernetes
@@ -269,6 +288,7 @@ jobs:
           ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }}_multi_cluster
 
       - name: Test Snapshot
+        if: ${{ matrix.os != 'windows-latest' }} # TODO: fix snapshot test on windows
         shell: bash
         run: |
           ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }}_snapshot
@@ -297,11 +317,13 @@ jobs:
           ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }}_restart
 
       - name: Test Port Forward
+        if: ${{ matrix.os != 'windows-latest' }} # TODO: fix port-forward on windows
         shell: bash
         run: |
           ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }}_port_forward
 
       - name: Test Exec
+        if: ${{ matrix.os != 'windows-latest' }} # TODO: fix exec on windows
         shell: bash
         run: |
           ./hack/e2e-test.sh kwokctl/kwokctl_${{ matrix.kwokctl-runtime }}_exec

pkg/config/config.go

@@ -455,7 +455,7 @@ func loadRaw(r io.Reader) ([]json.RawMessage, error) {
 			if errors.Is(err, io.EOF) {
 				break
 			}
-			return nil, err
+			return nil, fmt.Errorf("failed to decode %q: %w", raw, err)
 		}
 		if len(raw) == 0 {
 			// Ignore empty documents

pkg/kwokctl/runtime/binary/cluster.go

@@ -732,7 +732,7 @@ func (c *Cluster) Logs(ctx context.Context, name string, out io.Writer) error {
 
 	f, err := os.OpenFile(logs, os.O_RDONLY, 0640)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to open %s: %w", logs, err)
 	}
 	defer func() {
 		err = f.Close()

pkg/kwokctl/runtime/exec.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"path/filepath"
 	"strconv"
 	"strings"
 
@@ -36,7 +35,7 @@ import (
 // ForkExec forks a new process and execs the given command.
 // The process will be terminated when the context is canceled.
 func (c *Cluster) ForkExec(ctx context.Context, dir string, name string, args ...string) error {
-	pidPath := path.Join(dir, "pids", filepath.Base(name)+".pid")
+	pidPath := path.Join(dir, "pids", path.OnlyName(name)+".pid")
 	if file.Exists(pidPath) {
 		pidData, err := os.ReadFile(pidPath)
 		if err == nil {
@@ -50,7 +49,7 @@ func (c *Cluster) ForkExec(ctx context.Context, dir string, name string, args ..
 	}
 	ctx = exec.WithDir(ctx, dir)
 	ctx = exec.WithFork(ctx, true)
-	logPath := path.Join(dir, "logs", filepath.Base(name)+".log")
+	logPath := path.Join(dir, "logs", path.OnlyName(name)+".log")
 	logFile, err := c.OpenFile(logPath)
 	if err != nil {
 		return fmt.Errorf("open log file %s: %w", logPath, err)
@@ -80,7 +79,7 @@ func (c *Cluster) ForkExec(ctx context.Context, dir string, name string, args ..
 
 // ForkExecKill kills the process if it is running.
 func (c *Cluster) ForkExecKill(ctx context.Context, dir string, name string) error {
-	pidPath := path.Join(dir, "pids", filepath.Base(name)+".pid")
+	pidPath := path.Join(dir, "pids", path.OnlyName(name)+".pid")
 	if !file.Exists(pidPath) {
 		// No pid file exists, which means the process has been terminated
 		logger := log.FromContext(ctx)
@@ -116,7 +115,7 @@ func (c *Cluster) ForkExecKill(ctx context.Context, dir string, name string) err
 
 // ForkExecIsRunning checks if the process is running.
 func (c *Cluster) ForkExecIsRunning(ctx context.Context, dir string, name string) bool {
-	pidPath := path.Join(dir, "pids", filepath.Base(name)+".pid")
+	pidPath := path.Join(dir, "pids", path.OnlyName(name)+".pid")
 	if !file.Exists(pidPath) {
 		logger := log.FromContext(ctx)
 		logger.Debug("Stat file not exists",
@@ -206,7 +205,7 @@ func FormatExec(ctx context.Context, name string, args ...string) string {
 		_, _ = fmt.Fprintf(out, "%s ", strings.Join(opt.Env, " "))
 	}
 
-	_, _ = fmt.Fprintf(out, "%s", path.Base(name))
+	_, _ = fmt.Fprintf(out, "%s", path.OnlyName(name))
 
 	for _, arg := range args {
 		_, _ = fmt.Fprintf(out, " %s", arg)

pkg/utils/path/path.go

@@ -66,7 +66,11 @@ func Expand(path string) (string, error) {
 		}
 	}
 
-	return filepath.Abs(path)
+	p, err := filepath.Abs(path)
+	if err != nil {
+		return "", err
+	}
+	return Clean(p), nil
 }
 
 // RelFromHome returns a path relative to the home directory.
@@ -89,10 +93,20 @@ func Join(elem ...string) string {
 
 // Dir is a wrapper around filepath.Dir.
 func Dir(path string) string {
-	return filepath.Dir(path)
+	return Clean(filepath.Dir(path))
 }
 
 // Base is a wrapper around filepath.Base.
 func Base(path string) string {
 	return filepath.Base(path)
 }
+
+// Ext is a wrapper around filepath.Ext.
+func Ext(path string) string {
+	return filepath.Ext(path)
+}
+
+// OnlyName returns the file name without extension.
+func OnlyName(path string) string {
+	return strings.TrimSuffix(Base(path), Ext(path))
+}

pkg/utils/version/version_test.go

@@ -39,6 +39,12 @@ func TestParseFromOutput(t *testing.T) {
 			},
 			want: semver.MustParse("1.26.0"),
 		},
+		{
+			args: args{
+				s: "Kubernetes v0.0.0-master+$Format:%H$",
+			},
+			want: semver.MustParse("255.0.0"),
+		},
 		{
 			args: args{
 				s: "prometheus, version 2.35.0 (branch: HEAD)",

test/kwokctl/helper.sh

@@ -31,6 +31,12 @@ LOCAL_PATH="${ROOT_DIR}/bin/${GOOS}/${GOARCH}"
 
 export KWOK_CONTROLLER_BINARY="${LOCAL_PATH}/kwok"
 export KWOKCTL_CONTROLLER_BINARY="${LOCAL_PATH}/kwokctl"
+
+if [[ "${GOOS}" == "windows" ]]; then
+  KWOK_CONTROLLER_BINARY="${KWOK_CONTROLLER_BINARY}.exe"
+  KWOKCTL_CONTROLLER_BINARY="${KWOKCTL_CONTROLLER_BINARY}.exe"
+fi
+
 export KWOK_CONTROLLER_IMAGE="localhost/kwok:${VERSION}"
 export PATH="${LOCAL_PATH}:${PATH}"
 

test/kwokctl/kwokctl_workable_test.sh

@@ -74,11 +74,6 @@ function test_workable() {
     cat "${name}.kubeconfig"
     return 1
   fi
-
-  if ! kwokctl --name="${name}" etcdctl get /registry/namespaces/default --keys-only | grep default >/dev/null 2>&1; then
-    echo "Error: Failed to get namespace(default) by kwokctl etcdctl in cluster ${name}"
-    return 1
-  fi
 }
 
 function test_prometheus() {
@@ -122,6 +117,14 @@ function test_etcd_port() {
   fi
 }
 
+function test_etcdctl_get() {
+  local name="${1}"
+  if ! kwokctl --name="${name}" etcdctl get /registry/namespaces/default --keys-only | grep default >/dev/null 2>&1; then
+    echo "Error: Failed to get namespace(default) by kwokctl etcdctl in cluster ${name}"
+    return 1
+  fi
+}
+
 function test_kube_scheduler_port() {
   local result
 
@@ -180,6 +183,11 @@ function main() {
       test_kube_scheduler_port "${release}" || failed+=("kube_scheduler_port_${name}")
       test_etcd_port || failed+=("etcd_port_${name}")
     fi
+
+    # TODO: fix etcdctl get on windows
+    if [[ "${GOOS}" != "windows" ]]; then
+      test_etcdctl_get "${name}" || failed+=("etcdctl_get_${name}")
+    fi
     test_prometheus || failed+=("prometheus_${name}")
     test_kwok_controller_port || failed+=("kwok_controller_port_${name}")
     delete_cluster "${name}"

test/kwokctl/suite.sh

@@ -87,19 +87,18 @@ function retry() {
   done
 }
 
+GOOS="$(go env GOOS)"
+GOARCH="$(go env GOARCH)"
+
 function clear_testdata() {
   local name="${1}"
-  local arch
-  local os
-  arch="$(go env GOARCH)"
-  os="$(go env GOOS)"
 
   sed '/^ *$/d' |
     sed "s|${ROOT_DIR}|<ROOT_DIR>|g" |
     sed "s|${HOME}|~|g" |
     sed 's|/root/|~/|g' |
-    sed "s|${arch}|<ARCH>|g" |
-    sed "s|${os}|<OS>|g" |
+    sed "s|${GOARCH}|<ARCH>|g" |
+    sed "s|${GOOS}|<OS>|g" |
     sed "s|${name}|<CLUSTER_NAME>|g" |
     sed 's|\.tar\.gz|.<TAR>|g' |
     sed 's|\.zip|.<TAR>|g' |