Skip to content

Commit

Permalink
[kwokctl] Support kind-nerdctl as runtime
Browse files Browse the repository at this point in the history
  • Loading branch information
wzshiming committed Apr 8, 2024
1 parent 97b92c9 commit 7957923
Show file tree
Hide file tree
Showing 11 changed files with 285 additions and 13 deletions.
6 changes: 4 additions & 2 deletions .github/workflows/test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -154,12 +154,14 @@ jobs:
kwokctl-runtime: docker
- os: ubuntu-latest
kwokctl-runtime: podman
- os: ubuntu-latest
kwokctl-runtime: nerdctl
- os: ubuntu-latest
kwokctl-runtime: kind
- os: ubuntu-latest
kwokctl-runtime: kind-podman
- os: ubuntu-latest
kwokctl-runtime: nerdctl
kwokctl-runtime: kind-nerdctl

continue-on-error: false
runs-on: ${{ matrix.os }}
Expand Down Expand Up @@ -231,7 +233,7 @@ jobs:
docker info
- name: Install Nerdctl and Start Containerd
if: ${{ matrix.kwokctl-runtime == 'nerdctl' }}
if: ${{ matrix.kwokctl-runtime == 'nerdctl' || matrix.kwokctl-runtime == 'kind-nerdctl' }}
shell: bash
run: |
sudo mkdir -p /etc/cni/net.d
Expand Down
2 changes: 1 addition & 1 deletion hack/requirements.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ LOCAL_BIN_DIR="${ROOT_DIR}/bin"

export PATH="${LOCAL_BIN_DIR}:${PATH}"

KIND_VERSION=0.19.0
KIND_VERSION=0.22.0

KUBE_VERSION=1.29.0

Expand Down
15 changes: 8 additions & 7 deletions pkg/consts/consts.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ var (

EtcdBinaryPrefix = "https://github.com/etcd-io/etcd/releases/download"

KindVersion = "0.19.0"
KindVersion = "0.22.0"
KindBinaryPrefix = "https://github.com/kubernetes-sigs/kind/releases/download"
KindNodeImagePrefix = "docker.io/kindest"

Expand Down Expand Up @@ -68,12 +68,13 @@ var (

// The following runtime is provided.
const (
RuntimeTypeKind = "kind"
RuntimeTypeKindPodman = RuntimeTypeKind + "-" + RuntimeTypePodman
RuntimeTypeDocker = "docker"
RuntimeTypeNerdctl = "nerdctl"
RuntimeTypePodman = "podman"
RuntimeTypeBinary = "binary"
RuntimeTypeKind = "kind"
RuntimeTypeKindPodman = RuntimeTypeKind + "-" + RuntimeTypePodman
RuntimeTypeKindNerdctl = RuntimeTypeKind + "-" + RuntimeTypeNerdctl
RuntimeTypeDocker = "docker"
RuntimeTypeNerdctl = "nerdctl"
RuntimeTypePodman = "podman"
RuntimeTypeBinary = "binary"
)

// The following components is provided.
Expand Down
8 changes: 8 additions & 0 deletions pkg/kwokctl/runtime/kind/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,14 @@ func NewPodmanCluster(name, workdir string) (runtime.Runtime, error) {
}, nil
}

// NewNerdctlCluster creates a new Runtime for kind with nerdctl
func NewNerdctlCluster(name, workdir string) (runtime.Runtime, error) {
return &Cluster{
Cluster: runtime.NewCluster(name, workdir),
runtime: consts.RuntimeTypeNerdctl,
}, nil
}

// Available checks whether the runtime is available.
func (c *Cluster) Available(ctx context.Context) error {
if c.IsDryRun() {
Expand Down
1 change: 1 addition & 0 deletions pkg/kwokctl/runtime/kind/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,4 +24,5 @@ import (
func init() {
runtime.DefaultRegistry.Register(consts.RuntimeTypeKind, NewDockerCluster)
runtime.DefaultRegistry.Register(consts.RuntimeTypeKindPodman, NewPodmanCluster)
runtime.DefaultRegistry.Register(consts.RuntimeTypeKindNerdctl, NewNerdctlCluster)
}
4 changes: 2 additions & 2 deletions site/content/en/docs/generated/kwokctl_create_cluster.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ kwokctl create cluster [flags]
(default "docker.io/jaegertracing/all-in-one:1.53.0")
--jaeger-port uint32 Port to expose Jaeger UI
--kind-binary string Binary of kind, only for kind/kind-podman runtime
(default "https://github.com/kubernetes-sigs/kind/releases/download/v0.19.0/kind-linux-amd64")
(default "https://github.com/kubernetes-sigs/kind/releases/download/v0.22.0/kind-linux-amd64")
--kind-node-image string Image of kind node, only for kind/kind-podman runtime
'${KWOK_KIND_NODE_IMAGE_PREFIX}/node:${KWOK_KUBE_VERSION}'
(default "docker.io/kindest/node:v1.29.0")
Expand Down Expand Up @@ -80,7 +80,7 @@ kwokctl create cluster [flags]
(default "docker.io/prom/prometheus:v2.49.1")
--prometheus-port uint32 Port to expose Prometheus metrics
--quiet-pull Pull without printing progress information
--runtime string Runtime of the cluster (binary or docker or kind or kind-podman or nerdctl or podman)
--runtime string Runtime of the cluster (binary or docker or kind or kind-nerdctl or kind-podman or nerdctl or podman)
--secure-port The apiserver port on which to serve HTTPS with authentication and authorization, is not available before Kubernetes 1.13.0 (default true)
--timeout duration Timeout for waiting for the cluster to be created
--wait duration Wait for the cluster to be ready
Expand Down
2 changes: 1 addition & 1 deletion site/content/en/docs/generated/kwokctl_get_artifacts.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ kwokctl get artifacts [flags]
```
--filter string Filter the list of (binary or image)
-h, --help help for artifacts
--runtime string Runtime of the cluster (binary or docker or kind or kind-podman or nerdctl or podman)
--runtime string Runtime of the cluster (binary or docker or kind or kind-nerdctl or kind-podman or nerdctl or podman)
```

### Options inherited from parent commands
Expand Down
34 changes: 34 additions & 0 deletions test/e2e/kwokctl/kind-nerdctl/cluster_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kind_nerdctl_test

import (
"runtime"
"testing"

"sigs.k8s.io/kwok/test/e2e"
)

func TestMultiCluster(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("skipping test on non-linux platform")
}

f0 := e2e.CaseMultiCluster(kwokctlPath, logsDir, 1, baseArgs...).
Feature()
testEnv.Test(t, f0)
}
94 changes: 94 additions & 0 deletions test/e2e/kwokctl/kind-nerdctl/kubectl_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kind_nerdctl_test

import (
"runtime"
"testing"

"sigs.k8s.io/e2e-framework/pkg/envconf"

"sigs.k8s.io/kwok/test/e2e"
)

func TestHack(t *testing.T) {
f0 := e2e.CaseHack(kwokctlPath, clusterName, envconf.RandomName("node", 16)).
Feature()
testEnv.Test(t, f0)
}

func TestRecording(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("Skip test for non-linux platform")
}
f0 := e2e.CaseRecording(kwokctlPath, clusterName, pwd).
Feature()
testEnv.Test(t, f0)
}

func TestRecordingExternal(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("Skip test for non-linux platform")
}
f0 := e2e.CaseRecordingExternal(kwokctlPath, clusterName, pwd).
Feature()
testEnv.Test(t, f0)
}

func TestPortForward(t *testing.T) {
f0 := e2e.CasePortForward(kwokctlPath, clusterName, envconf.RandomName("node", 16), namespace).
Feature()
testEnv.Test(t, f0)
}

func TestLogs(t *testing.T) {
f0 := e2e.CaseLogs(kwokctlPath, clusterName, envconf.RandomName("node", 16), namespace, pwd).
Feature()
testEnv.Test(t, f0)
}

func TestAttach(t *testing.T) {
f0 := e2e.CaseAttach(kwokctlPath, clusterName, envconf.RandomName("node", 16), namespace, pwd).
Feature()
testEnv.Test(t, f0)
}

func TestExec(t *testing.T) {
f0 := e2e.CaseExec(kwokctlPath, clusterName, envconf.RandomName("node", 16), namespace).
Feature()
testEnv.Test(t, f0)
}

// Kind failed to start Load Kernel Modules
// https://github.com/kubernetes-sigs/kind/issues/3283
// func TestRestart(t *testing.T) {
// f0 := e2e.CaseRestart(kwokctlPath, clusterName).
// Feature()
// testEnv.Test(t, f0)
// }

func TestSnapshot(t *testing.T) {
f0 := e2e.CaseSnapshot(kwokctlPath, clusterName, pwd).
Feature()
testEnv.Test(t, f0)
}

func TestResourceUsage(t *testing.T) {
f0 := e2e.CaseResourceUsage(kwokctlPath, clusterName).
Feature()
testEnv.Test(t, f0)
}
37 changes: 37 additions & 0 deletions test/e2e/kwokctl/kind-nerdctl/kwok_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kind_nerdctl_test

import (
"testing"

"sigs.k8s.io/e2e-framework/pkg/envconf"

"sigs.k8s.io/kwok/test/e2e"
)

func TestNode(t *testing.T) {
f0 := e2e.CaseNode(envconf.RandomName("node", 16)).
Feature()
testEnv.Test(t, f0)
}

func TestPod(t *testing.T) {
f0 := e2e.CasePod(envconf.RandomName("node", 16), namespace).
Feature()
testEnv.Test(t, f0)
}
95 changes: 95 additions & 0 deletions test/e2e/kwokctl/kind-nerdctl/main_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
/*
Copyright 2023 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package kind_nerdctl_test is a test environment for kwok.
package kind_nerdctl_test

import (
"context"
"os"
"runtime"
"testing"

"sigs.k8s.io/e2e-framework/pkg/env"
"sigs.k8s.io/e2e-framework/pkg/envconf"
"sigs.k8s.io/e2e-framework/support/kwok"

"sigs.k8s.io/kwok/pkg/consts"
"sigs.k8s.io/kwok/pkg/log"
"sigs.k8s.io/kwok/pkg/utils/exec"
"sigs.k8s.io/kwok/pkg/utils/path"
"sigs.k8s.io/kwok/test/e2e/helper"
)

var (
runtimeEnv = consts.RuntimeTypeKindNerdctl
testEnv env.Environment
pwd = os.Getenv("PWD")
rootDir = path.Join(pwd, "../../../..")
logsDir = path.Join(rootDir, "logs")
clusterName = envconf.RandomName("kwok-e2e-kind-nerdctl", 16)
namespace = envconf.RandomName("ns", 16)
testImage = "localhost/kwok:test"
kwokctlPath = path.Join(rootDir, "bin", runtime.GOOS, runtime.GOARCH, "kwokctl"+helper.BinSuffix)
baseArgs = []string{
"--kwok-controller-image=" + testImage,
"--runtime=" + runtimeEnv,
"--enable-metrics-server",
"--wait=15m",
}
)

func init() {
_ = os.Setenv("KWOK_WORKDIR", path.Join(rootDir, "workdir"))
}

func TestMain(m *testing.M) {
testEnv = helper.Environment()

k := kwok.NewProvider().
WithName(clusterName).
WithPath(kwokctlPath)
testEnv.Setup(
helper.BuildKwokImage(rootDir, testImage, consts.RuntimeTypeNerdctl),
helper.BuildKwokctlBinary(rootDir),
func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
ctx, err := helper.CreateCluster(k, append(baseArgs,
"--config="+path.Join(rootDir, "test/e2e/port_forward.yaml"),
"--config="+path.Join(rootDir, "test/e2e/logs.yaml"),
"--config="+path.Join(rootDir, "test/e2e/attach.yaml"),
"--config="+path.Join(rootDir, "test/e2e/exec.yaml"),
"--config="+path.Join(rootDir, "kustomize/metrics/usage/usage-from-annotation.yaml"),
"--config="+path.Join(rootDir, "kustomize/metrics/resource/metrics-resource.yaml"),
)...)(ctx, cfg)
if err != nil {
logger := log.FromContext(ctx)

logger.Info("exporting logs for control plane")
err := exec.Exec(ctx, "docker", "logs", "kwok-"+clusterName+"-control-plane")
if err != nil {
logger.Error("failed to export logs for control plane", err)
}
}
return ctx, err
},
helper.CreateNamespace(namespace),
)
testEnv.Finish(
helper.ExportLogs(k, logsDir),
helper.DestroyCluster(k),
)
os.Exit(testEnv.Run(m))
}

0 comments on commit 7957923

Please sign in to comment.