From b0173e5476889903abfd2cd7c3336a80a1395f57 Mon Sep 17 00:00:00 2001 From: Neeraj Nagure Date: Mon, 17 Jun 2024 18:19:08 +0530 Subject: [PATCH] added a migration for dryrun test Signed-off-by: Neeraj Nagure --- .github/workflows/test.yaml | 6 + hack/update-all.sh | 5 + hack/update-testdata.sh | 28 + hack/verify-all.sh | 5 + hack/verify-testdata.sh | 31 + test/e2e/dryrun.go | 146 +++++ test/e2e/kwokctl/dryrun/dryrun_binary_test.go | 38 ++ test/e2e/kwokctl/dryrun/dryrun_docker_test.go | 38 ++ .../kwokctl/dryrun/dryrun_kind_podman_test.go | 38 ++ test/e2e/kwokctl/dryrun/dryrun_kind_test.go | 38 ++ .../e2e/kwokctl/dryrun/dryrun_nerdctl_test.go | 38 ++ test/e2e/kwokctl/dryrun/dryrun_podman_test.go | 38 ++ test/e2e/kwokctl/dryrun/main_test.go | 62 ++ .../dryrun/testdata/binary/create_cluster.txt | 45 ++ .../binary/create_cluster_with_extra.txt | 133 ++++ .../binary/create_cluster_with_verbosity.txt | 166 +++++ .../dryrun/testdata/docker/create_cluster.txt | 65 ++ .../docker/create_cluster_with_extra.txt | 154 +++++ .../docker/create_cluster_with_verbosity.txt | 194 ++++++ .../testdata/kind-podman/create_cluster.txt | 133 ++++ .../kind-podman/create_cluster_with_extra.txt | 341 ++++++++++ .../create_cluster_with_verbosity.txt | 582 ++++++++++++++++++ .../dryrun/testdata/kind/create_cluster.txt | 130 ++++ .../kind/create_cluster_with_extra.txt | 335 ++++++++++ .../kind/create_cluster_with_verbosity.txt | 564 +++++++++++++++++ .../testdata/nerdctl/create_cluster.txt | 65 ++ .../nerdctl/create_cluster_with_extra.txt | 154 +++++ .../nerdctl/create_cluster_with_verbosity.txt | 194 ++++++ .../dryrun/testdata/podman/create_cluster.txt | 65 ++ .../podman/create_cluster_with_extra.txt | 154 +++++ .../podman/create_cluster_with_verbosity.txt | 194 ++++++ 31 files changed, 4179 insertions(+) create mode 100755 hack/update-testdata.sh create mode 100755 hack/verify-testdata.sh create mode 100644 test/e2e/dryrun.go create mode 100644 test/e2e/kwokctl/dryrun/dryrun_binary_test.go create mode 100644 test/e2e/kwokctl/dryrun/dryrun_docker_test.go create mode 100644 test/e2e/kwokctl/dryrun/dryrun_kind_podman_test.go create mode 100644 test/e2e/kwokctl/dryrun/dryrun_kind_test.go create mode 100644 test/e2e/kwokctl/dryrun/dryrun_nerdctl_test.go create mode 100644 test/e2e/kwokctl/dryrun/dryrun_podman_test.go create mode 100644 test/e2e/kwokctl/dryrun/main_test.go create mode 100644 test/e2e/kwokctl/dryrun/testdata/binary/create_cluster.txt create mode 100644 test/e2e/kwokctl/dryrun/testdata/binary/create_cluster_with_extra.txt create mode 100644 test/e2e/kwokctl/dryrun/testdata/binary/create_cluster_with_verbosity.txt create mode 100644 test/e2e/kwokctl/dryrun/testdata/docker/create_cluster.txt create mode 100644 test/e2e/kwokctl/dryrun/testdata/docker/create_cluster_with_extra.txt create mode 100644 test/e2e/kwokctl/dryrun/testdata/docker/create_cluster_with_verbosity.txt create mode 100644 test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster.txt create mode 100644 test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster_with_extra.txt create mode 100644 test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster_with_verbosity.txt create mode 100644 test/e2e/kwokctl/dryrun/testdata/kind/create_cluster.txt create mode 100644 test/e2e/kwokctl/dryrun/testdata/kind/create_cluster_with_extra.txt create mode 100644 test/e2e/kwokctl/dryrun/testdata/kind/create_cluster_with_verbosity.txt create mode 100644 test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster.txt create mode 100644 test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster_with_extra.txt create mode 100644 test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster_with_verbosity.txt create mode 100644 test/e2e/kwokctl/dryrun/testdata/podman/create_cluster.txt create mode 100644 test/e2e/kwokctl/dryrun/testdata/podman/create_cluster_with_extra.txt create mode 100644 test/e2e/kwokctl/dryrun/testdata/podman/create_cluster_with_verbosity.txt diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ab0fab477..429da7505 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -258,6 +258,12 @@ jobs: run: | ./hack/e2e-test.sh kwokctl/kwokctl_with_dryrun + - name: Test e2e dry run + if: ${{ matrix.os == 'ubuntu-latest' && matrix.kwokctl-runtime == 'binary' }} + shell: bash + run: | + ./hack/e2e-test.sh e2e/kwokctl/dryrun + - name: Test e2e shell: bash run: | diff --git a/hack/update-all.sh b/hack/update-all.sh index b5de0cb78..0241382d7 100755 --- a/hack/update-all.sh +++ b/hack/update-all.sh @@ -88,6 +88,11 @@ if [[ "${UPDATE_HELM_CHARTS:-true}" == "true" ]]; then "${ROOT_DIR}"/hack/update-helm-charts.sh || failed+=(helm-charts) fi +if [[ "${UPDATE_DRY_RUN_TESTDATA:-true}" == "true" ]]; then + echo "[*] Update testdata..." + "${ROOT_DIR}"/hack/update-testdata.sh || failed+=(testdata) +fi + if [[ "${#failed[@]}" != 0 ]]; then echo "Update failed for: ${failed[*]}" exit 1 diff --git a/hack/update-testdata.sh b/hack/update-testdata.sh new file mode 100755 index 000000000..2e6581a52 --- /dev/null +++ b/hack/update-testdata.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +DIR="$(dirname "${BASH_SOURCE[0]}")" + +ROOT_DIR="$(realpath "${DIR}/..")" + +function update() { + go test -v ./test/e2e/kwokctl/dryrun -args --update-testdata +} + +cd "${ROOT_DIR}" && update diff --git a/hack/verify-all.sh b/hack/verify-all.sh index aa88551d9..f2cf6a276 100755 --- a/hack/verify-all.sh +++ b/hack/verify-all.sh @@ -108,6 +108,11 @@ if [[ "${VERIFY_DEPENDENCIES_VERSION:-true}" == "true" ]]; then "${ROOT_DIR}"/hack/verify-dependencies-version.sh || failed+=(dependencies-version) fi +if [[ "${VERIFY_DRY_RUN_TESTDATA:-true}" == "true" ]]; then + echo "[*] Verifying testdata..." + "${ROOT_DIR}"/hack/verify-testdata.sh || failed+=(testdata) +fi + # exit based on verify scripts if [[ "${#failed[@]}" != 0 ]]; then echo "Verify failed for: ${failed[*]}" diff --git a/hack/verify-testdata.sh b/hack/verify-testdata.sh new file mode 100755 index 000000000..3b115ef7a --- /dev/null +++ b/hack/verify-testdata.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +DIR="$(dirname "${BASH_SOURCE[0]}")" + +ROOT_DIR="$(realpath "${DIR}/..")" + +function check() { + if [[ "${UPDATE_DRY_RUN_TESTDATA:-"false"}" == "true" ]]; then + "${ROOT_DIR}"/hack/update-testdata.sh + fi + git --no-pager diff --exit-code +} + +cd "${ROOT_DIR}" && check diff --git a/test/e2e/dryrun.go b/test/e2e/dryrun.go new file mode 100644 index 000000000..2f0dfbc6a --- /dev/null +++ b/test/e2e/dryrun.go @@ -0,0 +1,146 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "io/fs" + "os" + "os/exec" + "regexp" + "runtime" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/features" + + "sigs.k8s.io/kwok/pkg/utils/path" +) + +var ( + emptyLine = regexp.MustCompile("\n{2,}") + homeDir, _ = os.UserHomeDir() +) + +func formatCmdOutput(got, clusterName, rootDir string) string { + got = strings.ReplaceAll(got, clusterName, "") + got = strings.ReplaceAll(got, rootDir, "") + got = strings.ReplaceAll(got, runtime.GOOS, "") + got = strings.ReplaceAll(got, runtime.GOARCH, "") + got = strings.ReplaceAll(got, ".zip", ".") + got = strings.ReplaceAll(got, ".tar.gz", ".") + got = strings.ReplaceAll(got, homeDir, "~") + got = emptyLine.ReplaceAllLiteralString(got, "\n") + return got +} + +func executeCommand(args []string, absPath string, clusterName string, kwokctlPath string, rootDir string, updateTestdata bool) (string, error) { + testdataPath := path.Join(rootDir, absPath) + expected, err := os.ReadFile(testdataPath) + if err != nil { + return "", err + } + cmd := exec.Command(kwokctlPath, args...) + output, err := cmd.Output() + if err != nil { + return "", err + } + want := string(expected) + got := string(output) + got = formatCmdOutput(got, clusterName, rootDir) + if diff := cmp.Diff(got, want); diff != "" { + if updateTestdata { + err = os.WriteFile(testdataPath, []byte(got), fs.FileMode(0644)) + if err != nil { + return "", err + } + } else { + return diff, nil + } + } + return "", nil +} + +func CaseDryrun(clusterName string, kwokctlPath string, rootDir string, clusterRuntime string, updateTestdata bool) *features.FeatureBuilder { + f := features.New("Dry run") + f = f.Assess("test cluster dryrun", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + absPath := "test/e2e/kwokctl/dryrun/testdata/" + clusterRuntime + "/create_cluster.txt" + args := []string{ + "create", "cluster", "--dry-run", "--name", clusterName, "--timeout=30m", + "--wait=30m", "--quiet-pull", "--disable-qps-limits", "--kube-authorization=false", + "--runtime", clusterRuntime, + } + diff, err := executeCommand(args, absPath, clusterName, kwokctlPath, rootDir, updateTestdata) + if err != nil { + t.Fatal(err) + } + if diff != "" { + t.Fatalf("Expected vs got:\n%s", diff) + } + return ctx + }) + return f +} + +func CaseDryrunWithExtra(clusterName string, kwokctlPath string, rootDir string, clusterRuntime string, updateTestdata bool) *features.FeatureBuilder { + f := features.New("Dry run with extra") + f = f.Assess("test cluster dryrun with extra", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + absPath := "test/e2e/kwokctl/dryrun/testdata/" + clusterRuntime + "/create_cluster_with_extra.txt" + extraPath := path.Join(rootDir, "test/kwokctl/testdata/extra.yaml") + args := []string{ + "create", "cluster", "--dry-run", "--name", clusterName, "--timeout=30m", + "--wait=30m", "--quiet-pull", "--disable-qps-limits", "--runtime", clusterRuntime, + "--config", extraPath, + } + diff, err := executeCommand(args, absPath, clusterName, kwokctlPath, rootDir, updateTestdata) + if err != nil { + t.Fatal(err) + } + if diff != "" { + t.Fatalf("Expected vs got:\n%s", diff) + } + return ctx + }) + return f +} + +func CaseDryrunWithVerbosity(clusterName string, kwokctlPath string, rootDir string, clusterRuntime string, updateTestdata bool) *features.FeatureBuilder { + f := features.New("Dry run with verbosity") + f = f.Assess("test cluster dryrun with verbosity", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + absPath := "test/e2e/kwokctl/dryrun/testdata/" + clusterRuntime + "/create_cluster_with_verbosity.txt" + kubeAuditPath := path.Join(rootDir, "test/kwokctl/audit-policy.yaml") + schedulerConfigPath := path.Join(rootDir, "test/kwokctl/scheduler-config.yaml") + args := []string{ + "create", "cluster", "--dry-run", "--name", clusterName, "--timeout=30m", "--wait=30m", + "--quiet-pull", "--disable-qps-limits", "--runtime", clusterRuntime, + "--prometheus-port=9090", "--jaeger-port=16686", "--dashboard-port=8000", + "--enable-metrics-server", "--kube-audit-policy", kubeAuditPath, + "--kube-scheduler-config", schedulerConfigPath, + } + diff, err := executeCommand(args, absPath, clusterName, kwokctlPath, rootDir, updateTestdata) + if err != nil { + t.Fatal(err) + } + if diff != "" { + t.Fatalf("Expected vs got:\n%s", diff) + } + return ctx + }) + return f +} diff --git a/test/e2e/kwokctl/dryrun/dryrun_binary_test.go b/test/e2e/kwokctl/dryrun/dryrun_binary_test.go new file mode 100644 index 000000000..8af255895 --- /dev/null +++ b/test/e2e/kwokctl/dryrun/dryrun_binary_test.go @@ -0,0 +1,38 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dryrun_test + +import ( + "testing" + + "sigs.k8s.io/kwok/test/e2e" +) + +func TestBinaryDryRun(t *testing.T) { + f0 := e2e.CaseDryrun(clusterName, kwokctlPath, rootDir, "binary", updateTestdata).Feature() + testEnv.Test(t, f0) +} + +func TestBinaryDryRunWithExtra(t *testing.T) { + f0 := e2e.CaseDryrunWithExtra(clusterName, kwokctlPath, rootDir, "binary", updateTestdata).Feature() + testEnv.Test(t, f0) +} + +func TestBinaryDryRunWithVerbosity(t *testing.T) { + f0 := e2e.CaseDryrunWithVerbosity(clusterName, kwokctlPath, rootDir, "binary", updateTestdata).Feature() + testEnv.Test(t, f0) +} diff --git a/test/e2e/kwokctl/dryrun/dryrun_docker_test.go b/test/e2e/kwokctl/dryrun/dryrun_docker_test.go new file mode 100644 index 000000000..494e38620 --- /dev/null +++ b/test/e2e/kwokctl/dryrun/dryrun_docker_test.go @@ -0,0 +1,38 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dryrun_test + +import ( + "testing" + + "sigs.k8s.io/kwok/test/e2e" +) + +func TestDockerDryRun(t *testing.T) { + f0 := e2e.CaseDryrun(clusterName, kwokctlPath, rootDir, "docker", updateTestdata).Feature() + testEnv.Test(t, f0) +} + +func TestDockerDryRunWithExtra(t *testing.T) { + f0 := e2e.CaseDryrunWithExtra(clusterName, kwokctlPath, rootDir, "docker", updateTestdata).Feature() + testEnv.Test(t, f0) +} + +func TestDockerDryRunWithVerbosity(t *testing.T) { + f0 := e2e.CaseDryrunWithVerbosity(clusterName, kwokctlPath, rootDir, "docker", updateTestdata).Feature() + testEnv.Test(t, f0) +} diff --git a/test/e2e/kwokctl/dryrun/dryrun_kind_podman_test.go b/test/e2e/kwokctl/dryrun/dryrun_kind_podman_test.go new file mode 100644 index 000000000..10e77b72c --- /dev/null +++ b/test/e2e/kwokctl/dryrun/dryrun_kind_podman_test.go @@ -0,0 +1,38 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dryrun_test + +import ( + "testing" + + "sigs.k8s.io/kwok/test/e2e" +) + +func TestKindPodmanDryRun(t *testing.T) { + f0 := e2e.CaseDryrun(clusterName, kwokctlPath, rootDir, "kind-podman", updateTestdata).Feature() + testEnv.Test(t, f0) +} + +func TestKindPodmanDryRunWithExtra(t *testing.T) { + f0 := e2e.CaseDryrunWithExtra(clusterName, kwokctlPath, rootDir, "kind-podman", updateTestdata).Feature() + testEnv.Test(t, f0) +} + +func TestKindPodmanDryRunWithVerbosity(t *testing.T) { + f0 := e2e.CaseDryrunWithVerbosity(clusterName, kwokctlPath, rootDir, "kind-podman", updateTestdata).Feature() + testEnv.Test(t, f0) +} diff --git a/test/e2e/kwokctl/dryrun/dryrun_kind_test.go b/test/e2e/kwokctl/dryrun/dryrun_kind_test.go new file mode 100644 index 000000000..35dd109bd --- /dev/null +++ b/test/e2e/kwokctl/dryrun/dryrun_kind_test.go @@ -0,0 +1,38 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dryrun_test + +import ( + "testing" + + "sigs.k8s.io/kwok/test/e2e" +) + +func TestKindDryRun(t *testing.T) { + f0 := e2e.CaseDryrun(clusterName, kwokctlPath, rootDir, "kind", updateTestdata).Feature() + testEnv.Test(t, f0) +} + +func TestKindDryRunWithExtra(t *testing.T) { + f0 := e2e.CaseDryrunWithExtra(clusterName, kwokctlPath, rootDir, "kind", updateTestdata).Feature() + testEnv.Test(t, f0) +} + +func TestKindDryRunWithVerbosity(t *testing.T) { + f0 := e2e.CaseDryrunWithVerbosity(clusterName, kwokctlPath, rootDir, "kind", updateTestdata).Feature() + testEnv.Test(t, f0) +} diff --git a/test/e2e/kwokctl/dryrun/dryrun_nerdctl_test.go b/test/e2e/kwokctl/dryrun/dryrun_nerdctl_test.go new file mode 100644 index 000000000..56ff2ae77 --- /dev/null +++ b/test/e2e/kwokctl/dryrun/dryrun_nerdctl_test.go @@ -0,0 +1,38 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dryrun_test + +import ( + "testing" + + "sigs.k8s.io/kwok/test/e2e" +) + +func TestNerdctlDryRun(t *testing.T) { + f0 := e2e.CaseDryrun(clusterName, kwokctlPath, rootDir, "nerdctl", updateTestdata).Feature() + testEnv.Test(t, f0) +} + +func TestNerdctlDryRunWithExtra(t *testing.T) { + f0 := e2e.CaseDryrunWithExtra(clusterName, kwokctlPath, rootDir, "nerdctl", updateTestdata).Feature() + testEnv.Test(t, f0) +} + +func TestNerdctlDryRunWithVerbosity(t *testing.T) { + f0 := e2e.CaseDryrunWithVerbosity(clusterName, kwokctlPath, rootDir, "nerdctl", updateTestdata).Feature() + testEnv.Test(t, f0) +} diff --git a/test/e2e/kwokctl/dryrun/dryrun_podman_test.go b/test/e2e/kwokctl/dryrun/dryrun_podman_test.go new file mode 100644 index 000000000..cb29213c7 --- /dev/null +++ b/test/e2e/kwokctl/dryrun/dryrun_podman_test.go @@ -0,0 +1,38 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dryrun_test + +import ( + "testing" + + "sigs.k8s.io/kwok/test/e2e" +) + +func TestPodmanDryRun(t *testing.T) { + f0 := e2e.CaseDryrun(clusterName, kwokctlPath, rootDir, "podman", updateTestdata).Feature() + testEnv.Test(t, f0) +} + +func TestPodmanDryRunWithExtra(t *testing.T) { + f0 := e2e.CaseDryrunWithExtra(clusterName, kwokctlPath, rootDir, "podman", updateTestdata).Feature() + testEnv.Test(t, f0) +} + +func TestPodmanDryRunWithVerbosity(t *testing.T) { + f0 := e2e.CaseDryrunWithVerbosity(clusterName, kwokctlPath, rootDir, "podman", updateTestdata).Feature() + testEnv.Test(t, f0) +} diff --git a/test/e2e/kwokctl/dryrun/main_test.go b/test/e2e/kwokctl/dryrun/main_test.go new file mode 100644 index 000000000..aba0747db --- /dev/null +++ b/test/e2e/kwokctl/dryrun/main_test.go @@ -0,0 +1,62 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package dryrun_test is a test environment for kwok. +package dryrun_test + +import ( + "flag" + "os" + "runtime" + "testing" + + "sigs.k8s.io/e2e-framework/pkg/env" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/support/kwok" + + "sigs.k8s.io/kwok/pkg/utils/path" + "sigs.k8s.io/kwok/test/e2e/helper" +) + +var ( + testEnv env.Environment + pwd = os.Getenv("PWD") + rootDir = path.Join(pwd, "../../../..") + logsDir = path.Join(rootDir, "logs") + clusterName = envconf.RandomName("kwok-e2e-dryrun", 16) + kwokctlPath = path.Join(rootDir, "bin", runtime.GOOS, runtime.GOARCH, "kwokctl"+helper.BinSuffix) + updateTestdata = false +) + +func init() { + _ = os.Setenv("KWOK_WORKDIR", path.Join(rootDir, "workdir")) + flag.BoolVar(&updateTestdata, "update-testdata", false, "update all of testdata") +} + +func TestMain(m *testing.M) { + testEnv = helper.Environment() + + k := kwok.NewProvider(). + WithName(clusterName). + WithPath(kwokctlPath) + testEnv.Setup( + helper.BuildKwokctlBinary(rootDir), + ) + testEnv.Finish( + helper.ExportLogs(k, logsDir), + ) + os.Exit(testEnv.Run(m)) +} diff --git a/test/e2e/kwokctl/dryrun/testdata/binary/create_cluster.txt b/test/e2e/kwokctl/dryrun/testdata/binary/create_cluster.txt new file mode 100644 index 000000000..e02b0f0f0 --- /dev/null +++ b/test/e2e/kwokctl/dryrun/testdata/binary/create_cluster.txt @@ -0,0 +1,45 @@ +# Save cluster config to /workdir/clusters//kwok.yaml +mkdir -p /workdir/clusters/ +mkdir -p /workdir/clusters//pids +mkdir -p /workdir/clusters//logs +mkdir -p /workdir/clusters//pki +# Generate PKI to /workdir/clusters//pki +mkdir -p /workdir/clusters//etcd +# Download https://github.com/etcd-io/etcd/releases/download/v3.5.11/etcd-v3.5.11--. and extract etcd to /workdir/clusters//bin/etcd +# Download https://dl.k8s.io/release/v1.30.2/bin///kube-apiserver to /workdir/clusters//bin/kube-apiserver +# Download https://dl.k8s.io/release/v1.30.2/bin///kube-controller-manager to /workdir/clusters//bin/kube-controller-manager +# Download https://dl.k8s.io/release/v1.30.2/bin///kube-scheduler to /workdir/clusters//bin/kube-scheduler +# Download https://github.com/kubernetes-sigs/kwok/releases/download/v0.6.0/kwok-- to /workdir/clusters//bin/kwok-controller +cat </workdir/clusters//kubeconfig.yaml +apiVersion: v1 +clusters: +- cluster: + certificate-authority: /workdir/clusters//pki/ca.crt + server: https://127.0.0.1:32764 + name: kwok- +contexts: +- context: + cluster: kwok- + user: kwok- + name: kwok- +current-context: kwok- +kind: Config +preferences: {} +users: +- name: kwok- + user: + client-certificate: /workdir/clusters//pki/admin.crt + client-key: /workdir/clusters//pki/admin.key +EOF +# Save cluster config to /workdir/clusters//kwok.yaml +# Add context kwok- to ~/.kube/config +cd /workdir/clusters/ && etcd --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/workdir/clusters//etcd --initial-advertise-peer-urls=http://0.0.0.0:32766 --listen-peer-urls=http://0.0.0.0:32766 --advertise-client-urls=http://0.0.0.0:32765 --listen-client-urls=http://0.0.0.0:32765 --initial-cluster=node0=http://0.0.0.0:32766 >/workdir/clusters//logs/etcd.log 2>&1 & +echo $! >/workdir/clusters//pids/etcd.pid +cd /workdir/clusters/ && kube-apiserver --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://127.0.0.1:32765 --bind-address=0.0.0.0 --secure-port=32764 --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --client-ca-file=/workdir/clusters//pki/ca.crt --service-account-key-file=/workdir/clusters//pki/admin.key --service-account-signing-key-file=/workdir/clusters//pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/workdir/clusters//pki/admin.key --proxy-client-cert-file=/workdir/clusters//pki/admin.crt >/workdir/clusters//logs/kube-apiserver.log 2>&1 & +echo $! >/workdir/clusters//pids/kube-apiserver.pid +cd /workdir/clusters/ && kube-controller-manager --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=/workdir/clusters//kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32762 --kube-api-qps=5000 --kube-api-burst=10000 >/workdir/clusters//logs/kube-controller-manager.log 2>&1 & +echo $! >/workdir/clusters//pids/kube-controller-manager.pid +cd /workdir/clusters/ && kube-scheduler --kubeconfig=/workdir/clusters//kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32761 --kube-api-qps=5000 --kube-api-burst=10000 >/workdir/clusters//logs/kube-scheduler.log 2>&1 & +echo $! >/workdir/clusters//pids/kube-scheduler.pid +cd /workdir/clusters/ && kwok-controller --manage-all-nodes=true --kubeconfig=/workdir/clusters//kubeconfig.yaml --config=/workdir/clusters//kwok.yaml --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --node-ip= --node-name=localhost --node-port=32763 --server-address=0.0.0.0:32763 --node-lease-duration-seconds=200 >/workdir/clusters//logs/kwok-controller.log 2>&1 & +echo $! >/workdir/clusters//pids/kwok-controller.pid diff --git a/test/e2e/kwokctl/dryrun/testdata/binary/create_cluster_with_extra.txt b/test/e2e/kwokctl/dryrun/testdata/binary/create_cluster_with_extra.txt new file mode 100644 index 000000000..249f4f243 --- /dev/null +++ b/test/e2e/kwokctl/dryrun/testdata/binary/create_cluster_with_extra.txt @@ -0,0 +1,133 @@ +# Save cluster config to /workdir/clusters//kwok.yaml +mkdir -p /workdir/clusters/ +mkdir -p /workdir/clusters//pids +mkdir -p /workdir/clusters//logs +mkdir -p /workdir/clusters//pki +# Generate PKI to /workdir/clusters//pki +mkdir -p /workdir/clusters//etcd +# Download https://github.com/etcd-io/etcd/releases/download/v3.5.11/etcd-v3.5.11--. and extract etcd to /workdir/clusters//bin/etcd +# Download https://dl.k8s.io/release/v1.30.2/bin///kube-apiserver to /workdir/clusters//bin/kube-apiserver +# Download https://dl.k8s.io/release/v1.30.2/bin///kube-controller-manager to /workdir/clusters//bin/kube-controller-manager +# Download https://dl.k8s.io/release/v1.30.2/bin///kube-scheduler to /workdir/clusters//bin/kube-scheduler +# Download https://github.com/kubernetes-sigs/kwok/releases/download/v0.6.0/kwok-- to /workdir/clusters//bin/kwok-controller +# Download https://github.com/prometheus/prometheus/releases/download/v2.53.0/prometheus-2.53.0.-. and extract prometheus to /workdir/clusters//bin/prometheus +cat </workdir/clusters//prometheus.yaml +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - follow_redirects: true + enable_http2: true + scheme: http + timeout: 10s + api_version: v2 + static_configs: + - targets: [] +scrape_configs: +- job_name: "etcd" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:32765 +- job_name: "kube-apiserver" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/workdir/clusters//pki/admin.crt" + key_file: "/workdir/clusters//pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:32764 +- job_name: "kube-controller-manager" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/workdir/clusters//pki/admin.crt" + key_file: "/workdir/clusters//pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:32762 +- job_name: "kube-scheduler" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/workdir/clusters//pki/admin.crt" + key_file: "/workdir/clusters//pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:32761 +- job_name: "kwok-controller-metrics-discovery" + http_sd_configs: + - url: http://127.0.0.1:32763/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:32763 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 +EOF +cat </workdir/clusters//kubeconfig.yaml +apiVersion: v1 +clusters: +- cluster: + certificate-authority: /workdir/clusters//pki/ca.crt + server: https://127.0.0.1:32764 + name: kwok- +contexts: +- context: + cluster: kwok- + user: kwok- + name: kwok- +current-context: kwok- +kind: Config +preferences: {} +users: +- name: kwok- + user: + client-certificate: /workdir/clusters//pki/admin.crt + client-key: /workdir/clusters//pki/admin.key +EOF +# Save cluster config to /workdir/clusters//kwok.yaml +# Add context kwok- to ~/.kube/config +cd /workdir/clusters/ && TEST_KEY=TEST_VALUE etcd --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/workdir/clusters//etcd --initial-advertise-peer-urls=http://0.0.0.0:32766 --listen-peer-urls=http://0.0.0.0:32766 --advertise-client-urls=http://0.0.0.0:32765 --listen-client-urls=http://0.0.0.0:32765 --initial-cluster=node0=http://0.0.0.0:32766 --log-level=debug >/workdir/clusters//logs/etcd.log 2>&1 & +echo $! >/workdir/clusters//pids/etcd.pid +cd /workdir/clusters/ && TEST_KEY=TEST_VALUE kube-apiserver --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://127.0.0.1:32765 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=32764 --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --client-ca-file=/workdir/clusters//pki/ca.crt --service-account-key-file=/workdir/clusters//pki/admin.key --service-account-signing-key-file=/workdir/clusters//pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/workdir/clusters//pki/admin.key --proxy-client-cert-file=/workdir/clusters//pki/admin.crt --v=5 >/workdir/clusters//logs/kube-apiserver.log 2>&1 & +echo $! >/workdir/clusters//pids/kube-apiserver.pid +cd /workdir/clusters/ && TEST_KEY=TEST_VALUE kube-controller-manager --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=/workdir/clusters//kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32762 --root-ca-file=/workdir/clusters//pki/ca.crt --service-account-private-key-file=/workdir/clusters//pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=5 >/workdir/clusters//logs/kube-controller-manager.log 2>&1 & +echo $! >/workdir/clusters//pids/kube-controller-manager.pid +cd /workdir/clusters/ && TEST_KEY=TEST_VALUE kube-scheduler --kubeconfig=/workdir/clusters//kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32761 --kube-api-qps=5000 --kube-api-burst=10000 --v=5 >/workdir/clusters//logs/kube-scheduler.log 2>&1 & +echo $! >/workdir/clusters//pids/kube-scheduler.pid +cd /workdir/clusters/ && TEST_KEY=TEST_VALUE kwok-controller --manage-all-nodes=true --kubeconfig=/workdir/clusters//kubeconfig.yaml --config=/workdir/clusters//kwok.yaml --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --node-ip= --node-name=localhost --node-port=32763 --server-address=0.0.0.0:32763 --node-lease-duration-seconds=200 --v=-4 >/workdir/clusters//logs/kwok-controller.log 2>&1 & +echo $! >/workdir/clusters//pids/kwok-controller.pid +cd /workdir/clusters/ && TEST_KEY=TEST_VALUE prometheus --config.file=/workdir/clusters//prometheus.yaml --web.listen-address=0.0.0.0:9090 --log.level=debug >/workdir/clusters//logs/prometheus.log 2>&1 & +echo $! >/workdir/clusters//pids/prometheus.pid diff --git a/test/e2e/kwokctl/dryrun/testdata/binary/create_cluster_with_verbosity.txt b/test/e2e/kwokctl/dryrun/testdata/binary/create_cluster_with_verbosity.txt new file mode 100644 index 000000000..c0d65283f --- /dev/null +++ b/test/e2e/kwokctl/dryrun/testdata/binary/create_cluster_with_verbosity.txt @@ -0,0 +1,166 @@ +# Save cluster config to /workdir/clusters//kwok.yaml +mkdir -p /workdir/clusters/ +mkdir -p /workdir/clusters//pids +mkdir -p /workdir/clusters//logs +mkdir -p /workdir/clusters//pki +# Generate PKI to /workdir/clusters//pki +touch /workdir/clusters//logs/audit.log +cp /test/kwokctl/audit-policy.yaml /workdir/clusters//audit.yaml +mkdir -p /workdir/clusters//etcd +# Download https://github.com/etcd-io/etcd/releases/download/v3.5.11/etcd-v3.5.11--. and extract etcd to /workdir/clusters//bin/etcd +# Download https://dl.k8s.io/release/v1.30.2/bin///kube-apiserver to /workdir/clusters//bin/kube-apiserver +cat </workdir/clusters//apiserver-tracing-config.yaml +apiVersion: apiserver.config.k8s.io/v1alpha1 +kind: TracingConfiguration +endpoint: 127.0.0.1:32762 +samplingRatePerMillion: 1000000 +EOF +# Download https://dl.k8s.io/release/v1.30.2/bin///kube-controller-manager to /workdir/clusters//bin/kube-controller-manager +# Download https://dl.k8s.io/release/v1.30.2/bin///kube-scheduler to /workdir/clusters//bin/kube-scheduler +cp /test/kwokctl/scheduler-config.yaml /workdir/clusters//scheduler.yaml +cat <>/workdir/clusters//scheduler.yaml +clientConnection: + kubeconfig: "/workdir/clusters//kubeconfig.yaml" +EOF +# Download https://github.com/kubernetes-sigs/kwok/releases/download/v0.6.0/kwok-- to /workdir/clusters//bin/kwok-controller +# Download https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.7.1/metrics-server-- to /workdir/clusters//bin/metrics-server +# Download https://github.com/prometheus/prometheus/releases/download/v2.53.0/prometheus-2.53.0.-. and extract prometheus to /workdir/clusters//bin/prometheus +# Download https://github.com/jaegertracing/jaeger/releases/download/v1.58.1/jaeger-1.58.1--. and extract jaeger-all-in-one to /workdir/clusters//bin/jaeger +cat </workdir/clusters//prometheus.yaml +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - follow_redirects: true + enable_http2: true + scheme: http + timeout: 10s + api_version: v2 + static_configs: + - targets: [] +scrape_configs: +- job_name: "etcd" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:32765 +- job_name: "kube-apiserver" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/workdir/clusters//pki/admin.crt" + key_file: "/workdir/clusters//pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:32764 +- job_name: "kube-controller-manager" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/workdir/clusters//pki/admin.crt" + key_file: "/workdir/clusters//pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:32761 +- job_name: "kube-scheduler" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/workdir/clusters//pki/admin.crt" + key_file: "/workdir/clusters//pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:32760 +- job_name: "kwok-controller-metrics-discovery" + http_sd_configs: + - url: http://127.0.0.1:32763/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:32763 +- job_name: "metrics-server" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/workdir/clusters//pki/admin.crt" + key_file: "/workdir/clusters//pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:32759 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 +EOF +cat </workdir/clusters//kubeconfig.yaml +apiVersion: v1 +clusters: +- cluster: + certificate-authority: /workdir/clusters//pki/ca.crt + server: https://127.0.0.1:32764 + name: kwok- +contexts: +- context: + cluster: kwok- + user: kwok- + name: kwok- +current-context: kwok- +kind: Config +preferences: {} +users: +- name: kwok- + user: + client-certificate: /workdir/clusters//pki/admin.crt + client-key: /workdir/clusters//pki/admin.key +EOF +# Save cluster config to /workdir/clusters//kwok.yaml +# Add context kwok- to ~/.kube/config +cd /workdir/clusters/ && etcd --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/workdir/clusters//etcd --initial-advertise-peer-urls=http://0.0.0.0:32766 --listen-peer-urls=http://0.0.0.0:32766 --advertise-client-urls=http://0.0.0.0:32765 --listen-client-urls=http://0.0.0.0:32765 --initial-cluster=node0=http://0.0.0.0:32766 >/workdir/clusters//logs/etcd.log 2>&1 & +echo $! >/workdir/clusters//pids/etcd.pid +cd /workdir/clusters/ && jaeger --collector.otlp.enabled=true --query.http-server.host-port=0.0.0.0:16686 --collector.otlp.grpc.host-port=127.0.0.1:32762 >/workdir/clusters//logs/jaeger.log 2>&1 & +echo $! >/workdir/clusters//pids/jaeger.pid +cd /workdir/clusters/ && kube-apiserver --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://127.0.0.1:32765 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=32764 --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --client-ca-file=/workdir/clusters//pki/ca.crt --service-account-key-file=/workdir/clusters//pki/admin.key --service-account-signing-key-file=/workdir/clusters//pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/workdir/clusters//pki/admin.key --proxy-client-cert-file=/workdir/clusters//pki/admin.crt --audit-policy-file=/workdir/clusters//audit.yaml --audit-log-path=/workdir/clusters//logs/audit.log --tracing-config-file=/workdir/clusters//apiserver-tracing-config.yaml >/workdir/clusters//logs/kube-apiserver.log 2>&1 & +echo $! >/workdir/clusters//pids/kube-apiserver.pid +cd /workdir/clusters/ && kube-controller-manager --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=/workdir/clusters//kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32761 --root-ca-file=/workdir/clusters//pki/ca.crt --service-account-private-key-file=/workdir/clusters//pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 >/workdir/clusters//logs/kube-controller-manager.log 2>&1 & +echo $! >/workdir/clusters//pids/kube-controller-manager.pid +cd /workdir/clusters/ && kube-scheduler --config=/workdir/clusters//scheduler.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32760 --kube-api-qps=5000 --kube-api-burst=10000 >/workdir/clusters//logs/kube-scheduler.log 2>&1 & +echo $! >/workdir/clusters//pids/kube-scheduler.pid +cd /workdir/clusters/ && kwok-controller --manage-all-nodes=true --kubeconfig=/workdir/clusters//kubeconfig.yaml --config=/workdir/clusters//kwok.yaml --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key --node-ip= --node-name=localhost --node-port=32763 --server-address=0.0.0.0:32763 --node-lease-duration-seconds=200 >/workdir/clusters//logs/kwok-controller.log 2>&1 & +echo $! >/workdir/clusters//pids/kwok-controller.pid +cd /workdir/clusters/ && metrics-server --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --kubelet-use-node-status-port --kubelet-insecure-tls --metric-resolution=15s --bind-address=0.0.0.0 --secure-port=32759 --kubeconfig=/workdir/clusters//kubeconfig.yaml --authentication-kubeconfig=/workdir/clusters//kubeconfig.yaml --authorization-kubeconfig=/workdir/clusters//kubeconfig.yaml --tls-cert-file=/workdir/clusters//pki/admin.crt --tls-private-key-file=/workdir/clusters//pki/admin.key >/workdir/clusters//logs/metrics-server.log 2>&1 & +echo $! >/workdir/clusters//pids/metrics-server.pid +cd /workdir/clusters/ && prometheus --config.file=/workdir/clusters//prometheus.yaml --web.listen-address=0.0.0.0:9090 >/workdir/clusters//logs/prometheus.log 2>&1 & +echo $! >/workdir/clusters//pids/prometheus.pid +# Set up apiservice for metrics server diff --git a/test/e2e/kwokctl/dryrun/testdata/docker/create_cluster.txt b/test/e2e/kwokctl/dryrun/testdata/docker/create_cluster.txt new file mode 100644 index 000000000..3c33c5e75 --- /dev/null +++ b/test/e2e/kwokctl/dryrun/testdata/docker/create_cluster.txt @@ -0,0 +1,65 @@ +# Save cluster config to /workdir/clusters//kwok.yaml +mkdir -p /workdir/clusters/ +mkdir -p /workdir/clusters//pki +# Generate PKI to /workdir/clusters//pki +mkdir -p /workdir/clusters//etcd +docker pull registry.k8s.io/etcd:3.5.11-0 +docker pull registry.k8s.io/kube-apiserver:v1.30.2 +docker pull registry.k8s.io/kube-controller-manager:v1.30.2 +docker pull registry.k8s.io/kube-scheduler:v1.30.2 +docker pull registry.k8s.io/kwok/kwok:v0.6.0 +cat </workdir/clusters//kubeconfig.yaml +apiVersion: v1 +clusters: +- cluster: + certificate-authority: /workdir/clusters//pki/ca.crt + server: https://127.0.0.1:32766 + name: kwok- +contexts: +- context: + cluster: kwok- + user: kwok- + name: kwok- +current-context: kwok- +kind: Config +preferences: {} +users: +- name: kwok- + user: + client-certificate: /workdir/clusters//pki/admin.crt + client-key: /workdir/clusters//pki/admin.key +EOF +cat </workdir/clusters//kubeconfig +apiVersion: v1 +clusters: +- cluster: + certificate-authority: /etc/kubernetes/pki/ca.crt + server: https://kwok--kube-apiserver:6443 + name: kwok- +contexts: +- context: + cluster: kwok- + user: kwok- + name: kwok- +current-context: kwok- +kind: Config +preferences: {} +users: +- name: kwok- + user: + client-certificate: /etc/kubernetes/pki/admin.crt + client-key: /etc/kubernetes/pki/admin.key +EOF +# Save cluster config to /workdir/clusters//kwok.yaml +docker network create kwok- --label=com.docker.compose.project=kwok- +docker create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 +docker create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --link=kwok--etcd --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-apiserver:v1.30.2 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt +docker create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.30.2 --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=/root/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --kube-api-qps=5000 --kube-api-burst=10000 +docker create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-scheduler:v1.30.2 --kubeconfig=/root/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 +docker create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:/root/.kwok/kwok.yaml:ro registry.k8s.io/kwok/kwok:v0.6.0 --manage-all-nodes=true --kubeconfig=/root/.kube/config --config=/root/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=200 +# Add context kwok- to ~/.kube/config +docker start kwok--etcd +docker start kwok--kube-apiserver +docker start kwok--kube-controller-manager +docker start kwok--kube-scheduler +docker start kwok--kwok-controller diff --git a/test/e2e/kwokctl/dryrun/testdata/docker/create_cluster_with_extra.txt b/test/e2e/kwokctl/dryrun/testdata/docker/create_cluster_with_extra.txt new file mode 100644 index 000000000..ddc46803e --- /dev/null +++ b/test/e2e/kwokctl/dryrun/testdata/docker/create_cluster_with_extra.txt @@ -0,0 +1,154 @@ +# Save cluster config to /workdir/clusters//kwok.yaml +mkdir -p /workdir/clusters/ +mkdir -p /workdir/clusters//pki +# Generate PKI to /workdir/clusters//pki +mkdir -p /workdir/clusters//etcd +docker pull registry.k8s.io/etcd:3.5.11-0 +docker pull registry.k8s.io/kube-apiserver:v1.30.2 +docker pull registry.k8s.io/kube-controller-manager:v1.30.2 +docker pull registry.k8s.io/kube-scheduler:v1.30.2 +docker pull registry.k8s.io/kwok/kwok:v0.6.0 +docker pull docker.io/prom/prometheus:v2.53.0 +cat </workdir/clusters//prometheus.yaml +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - follow_redirects: true + enable_http2: true + scheme: http + timeout: 10s + api_version: v2 + static_configs: + - targets: [] +scrape_configs: +- job_name: "etcd" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - kwok--etcd:2379 +- job_name: "kube-apiserver" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - kwok--kube-apiserver:6443 +- job_name: "kube-controller-manager" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - kwok--kube-controller-manager:10257 +- job_name: "kube-scheduler" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - kwok--kube-scheduler:10259 +- job_name: "kwok-controller-metrics-discovery" + http_sd_configs: + - url: http://kwok--kwok-controller:10247/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - kwok--kwok-controller:10247 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 +EOF +chmod 0644 /workdir/clusters//prometheus.yaml +cat </workdir/clusters//kubeconfig.yaml +apiVersion: v1 +clusters: +- cluster: + certificate-authority: /workdir/clusters//pki/ca.crt + server: https://127.0.0.1:32766 + name: kwok- +contexts: +- context: + cluster: kwok- + user: kwok- + name: kwok- +current-context: kwok- +kind: Config +preferences: {} +users: +- name: kwok- + user: + client-certificate: /workdir/clusters//pki/admin.crt + client-key: /workdir/clusters//pki/admin.key +EOF +cat </workdir/clusters//kubeconfig +apiVersion: v1 +clusters: +- cluster: + certificate-authority: /etc/kubernetes/pki/ca.crt + server: https://kwok--kube-apiserver:6443 + name: kwok- +contexts: +- context: + cluster: kwok- + user: kwok- + name: kwok- +current-context: kwok- +kind: Config +preferences: {} +users: +- name: kwok- + user: + client-certificate: /etc/kubernetes/pki/admin.crt + client-key: /etc/kubernetes/pki/admin.key +EOF +# Save cluster config to /workdir/clusters//kwok.yaml +docker network create kwok- --label=com.docker.compose.project=kwok- +docker create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug +docker create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --link=kwok--etcd --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/test/e2e/kwokctl/dryrun/extras/apiserver:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.30.2 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt --v=5 +docker create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/test/e2e/kwokctl/dryrun/extras/controller-manager:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.30.2 --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=/root/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=5 +docker create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/test/e2e/kwokctl/dryrun/extras/scheduler:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.30.2 --kubeconfig=/root/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=5 +docker create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:/root/.kwok/kwok.yaml:ro --volume=/test/e2e/kwokctl/dryrun/extras/controller:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kwok/kwok:v0.6.0 --manage-all-nodes=true --kubeconfig=/root/.kube/config --config=/root/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=200 --v=-4 +docker create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --link=kwok--etcd --link=kwok--kube-apiserver --link=kwok--kube-controller-manager --link=kwok--kube-scheduler --link=kwok--kwok-controller --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/test/e2e/kwokctl/dryrun/extras/prometheus:/extras/tmp --env=TEST_KEY=TEST_VALUE docker.io/prom/prometheus:v2.53.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 --log.level=debug +# Add context kwok- to ~/.kube/config +docker start kwok--etcd +docker start kwok--kube-apiserver +docker start kwok--kube-controller-manager +docker start kwok--kube-scheduler +docker start kwok--kwok-controller +docker start kwok--prometheus diff --git a/test/e2e/kwokctl/dryrun/testdata/docker/create_cluster_with_verbosity.txt b/test/e2e/kwokctl/dryrun/testdata/docker/create_cluster_with_verbosity.txt new file mode 100644 index 000000000..1afdb8d46 --- /dev/null +++ b/test/e2e/kwokctl/dryrun/testdata/docker/create_cluster_with_verbosity.txt @@ -0,0 +1,194 @@ +# Save cluster config to /workdir/clusters//kwok.yaml +mkdir -p /workdir/clusters/ +mkdir -p /workdir/clusters//pki +# Generate PKI to /workdir/clusters//pki +mkdir -p /workdir/clusters//logs +touch /workdir/clusters//logs/audit.log +cp /test/kwokctl/audit-policy.yaml /workdir/clusters//audit.yaml +mkdir -p /workdir/clusters//etcd +docker pull registry.k8s.io/etcd:3.5.11-0 +docker pull registry.k8s.io/kube-apiserver:v1.30.2 +cat </workdir/clusters//apiserver-tracing-config.yaml +apiVersion: apiserver.config.k8s.io/v1alpha1 +kind: TracingConfiguration +endpoint: kwok--jaeger:4317 +samplingRatePerMillion: 1000000 +EOF +docker pull registry.k8s.io/kube-controller-manager:v1.30.2 +cp /test/kwokctl/scheduler-config.yaml /workdir/clusters//scheduler.yaml +cat <>/workdir/clusters//scheduler.yaml +clientConnection: + kubeconfig: "/root/.kube/config" +EOF +docker pull registry.k8s.io/kube-scheduler:v1.30.2 +docker pull registry.k8s.io/kwok/kwok:v0.6.0 +docker pull registry.k8s.io/metrics-server/metrics-server:v0.7.1 +docker pull docker.io/prom/prometheus:v2.53.0 +docker pull docker.io/jaegertracing/all-in-one:1.58.1 +docker pull docker.io/kubernetesui/dashboard:v2.7.0 +docker pull docker.io/kubernetesui/metrics-scraper:v1.0.9 +cat </workdir/clusters//prometheus.yaml +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - follow_redirects: true + enable_http2: true + scheme: http + timeout: 10s + api_version: v2 + static_configs: + - targets: [] +scrape_configs: +- job_name: "etcd" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - kwok--etcd:2379 +- job_name: "kube-apiserver" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - kwok--kube-apiserver:6443 +- job_name: "kube-controller-manager" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - kwok--kube-controller-manager:10257 +- job_name: "kube-scheduler" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - kwok--kube-scheduler:10259 +- job_name: "kwok-controller-metrics-discovery" + http_sd_configs: + - url: http://kwok--kwok-controller:10247/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - kwok--kwok-controller:10247 +- job_name: "metrics-server" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - kwok--kwok-controller:4443 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 +EOF +chmod 0644 /workdir/clusters//prometheus.yaml +cat </workdir/clusters//kubeconfig.yaml +apiVersion: v1 +clusters: +- cluster: + certificate-authority: /workdir/clusters//pki/ca.crt + server: https://127.0.0.1:32766 + name: kwok- +contexts: +- context: + cluster: kwok- + user: kwok- + name: kwok- +current-context: kwok- +kind: Config +preferences: {} +users: +- name: kwok- + user: + client-certificate: /workdir/clusters//pki/admin.crt + client-key: /workdir/clusters//pki/admin.key +EOF +cat </workdir/clusters//kubeconfig +apiVersion: v1 +clusters: +- cluster: + certificate-authority: /etc/kubernetes/pki/ca.crt + server: https://kwok--kube-apiserver:6443 + name: kwok- +contexts: +- context: + cluster: kwok- + user: kwok- + name: kwok- +current-context: kwok- +kind: Config +preferences: {} +users: +- name: kwok- + user: + client-certificate: /etc/kubernetes/pki/admin.crt + client-key: /etc/kubernetes/pki/admin.key +EOF +# Save cluster config to /workdir/clusters//kwok.yaml +docker network create kwok- --label=com.docker.compose.project=kwok- +docker create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 +docker create --name=kwok--jaeger --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=16686:16686/tcp docker.io/jaegertracing/all-in-one:1.58.1 --collector.otlp.enabled=true --query.http-server.host-port=0.0.0.0:16686 +docker create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --link=kwok--etcd --link=kwok--jaeger --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//audit.yaml:/etc/kubernetes/audit-policy.yaml:ro --volume=/workdir/clusters//logs/audit.log:/var/log/kubernetes/audit/audit.log --volume=/workdir/clusters//apiserver-tracing-config.yaml:/etc/kubernetes/apiserver-tracing-config.yaml:ro registry.k8s.io/kube-apiserver:v1.30.2 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt --audit-policy-file=/etc/kubernetes/audit-policy.yaml --audit-log-path=/var/log/kubernetes/audit/audit.log --tracing-config-file=/etc/kubernetes/apiserver-tracing-config.yaml +docker create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.30.2 --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=/root/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 +docker create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//scheduler.yaml:/etc/kubernetes/scheduler.yaml:ro registry.k8s.io/kube-scheduler:v1.30.2 --config=/etc/kubernetes/scheduler.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 +docker create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:/root/.kwok/kwok.yaml:ro registry.k8s.io/kwok/kwok:v0.6.0 --manage-all-nodes=true --kubeconfig=/root/.kube/config --config=/root/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=200 +docker create --name=kwok--dashboard --pull=never --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=8000:8080/tcp --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/kubernetesui/dashboard:v2.7.0 --insecure-bind-address=0.0.0.0 --bind-address=127.0.0.1 --port=0 --enable-insecure-login --enable-skip-login --disable-settings-authorizer --sidecar-host=kwok--dashboard-metrics-scraper:8000 --system-banner=Welcome to kwok- --kubeconfig=/root/.kube/config --insecure-port=8080 +docker create --name=kwok--metrics-server --pull=never --entrypoint=/metrics-server --network=kwok- --user=root --link=kwok--kwok-controller --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/metrics-server/metrics-server:v0.7.1 --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --kubelet-use-node-status-port --kubelet-insecure-tls --metric-resolution=15s --bind-address=0.0.0.0 --secure-port=4443 --kubeconfig=/root/.kube/config --authentication-kubeconfig=/root/.kube/config --authorization-kubeconfig=/root/.kube/config --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key +docker create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --link=kwok--etcd --link=kwok--kube-apiserver --link=kwok--kube-controller-manager --link=kwok--kube-scheduler --link=kwok--kwok-controller --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/prom/prometheus:v2.53.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 +docker create --name=kwok--dashboard-metrics-scraper --pull=never --network=kwok- --user=root --link=kwok--metrics-server --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/kubernetesui/metrics-scraper:v1.0.9 --db-file=/metrics.db --kubeconfig=/root/.kube/config +# Add context kwok- to ~/.kube/config +docker start kwok--etcd +docker start kwok--jaeger +docker start kwok--kube-apiserver +docker start kwok--kube-controller-manager +docker start kwok--kube-scheduler +docker start kwok--kwok-controller +docker start kwok--dashboard +docker start kwok--metrics-server +docker start kwok--prometheus +docker start kwok--dashboard-metrics-scraper +# Set up apiservice for metrics server diff --git a/test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster.txt b/test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster.txt new file mode 100644 index 000000000..f0e39669c --- /dev/null +++ b/test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster.txt @@ -0,0 +1,133 @@ +# Save cluster config to /workdir/clusters//kwok.yaml +mkdir -p /workdir/clusters/ +mkdir -p /workdir/clusters//pki +# Generate PKI to /workdir/clusters//pki +mkdir -p /workdir/clusters//pki/etcd +podman pull docker.io/kindest/node:v1.30.2 +mkdir -p /workdir/clusters//pki +mkdir -p /workdir/clusters//manifests +cat </workdir/clusters//kind.yaml +apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +kubeadmConfigPatches: +- | + apiServer: + extraArgs: + enable-priority-and-fairness: "false" + max-mutating-requests-inflight: "0" + max-requests-inflight: "0" + apiVersion: kubeadm.k8s.io/v1beta3 + controllerManager: + extraArgs: + kube-api-burst: "10000" + kube-api-qps: "5000" + dns: {} + etcd: + local: + dataDir: /var/lib/etcd + kind: ClusterConfiguration + networking: {} + scheduler: + extraArgs: + kube-api-burst: "10000" + kube-api-qps: "5000" +networking: + apiServerPort: 32766 +nodes: +- extraMounts: + - containerPath: /etc/kwok/ + hostPath: /workdir/clusters/ + - containerPath: /etc/kubernetes/manifests + hostPath: /workdir/clusters//manifests + - containerPath: /etc/kubernetes/pki + hostPath: /workdir/clusters//pki + extraPortMappings: + - containerPort: 2379 + hostPort: 32765 + protocol: TCP + role: control-plane +EOF +podman pull registry.k8s.io/kwok/kwok:v0.6.0 +cat </workdir/clusters//manifests/kwok-controller.yaml +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: kwok-controller + namespace: kube-system +spec: + containers: + - args: + - --manage-all-nodes=false + - --manage-nodes-with-annotation-selector=kwok.x-k8s.io/node=fake + - --kubeconfig=/root/.kube/config + - --config=/root/.kwok/kwok.yaml + - --tls-cert-file=/etc/kubernetes/pki/admin.crt + - --tls-private-key-file=/etc/kubernetes/pki/admin.key + - --node-ip=$(POD_IP) + - --node-name=kwok-controller.kube-system.svc + - --node-port=10247 + - --server-address=0.0.0.0:10247 + - --node-lease-duration-seconds=40 + command: + - kwok + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: registry.k8s.io/kwok/kwok:v0.6.0 + imagePullPolicy: Never + name: kwok-controller + resources: {} + volumeMounts: + - mountPath: /root/.kube/config + name: volume-0 + readOnly: true + - mountPath: /etc/kubernetes/pki/ca.crt + name: volume-1 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.crt + name: volume-2 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.key + name: volume-3 + readOnly: true + - mountPath: /root/.kwok/kwok.yaml + name: volume-4 + readOnly: true + hostNetwork: true + restartPolicy: Always + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: volume-0 + - hostPath: + path: /etc/kubernetes/pki/ca.crt + name: volume-1 + - hostPath: + path: /etc/kubernetes/pki/admin.crt + name: volume-2 + - hostPath: + path: /etc/kubernetes/pki/admin.key + name: volume-3 + - hostPath: + path: /etc/kwok/kwok.yaml + name: volume-4 +status: {} +EOF +# Save cluster config to /workdir/clusters//kwok.yaml +KIND_EXPERIMENTAL_PROVIDER=podman kind create cluster --config /workdir/clusters//kind.yaml --name kwok- --image docker.io/kindest/node:v1.30.2 --wait 29m +mkdir -p /workdir/cache/image-archive/registry.k8s.io/kwok/kwok +podman save registry.k8s.io/kwok/kwok:v0.6.0 -o /workdir/cache/image-archive/registry.k8s.io/kwok/kwok/v0.6.0.tar +KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/registry.k8s.io/kwok/kwok/v0.6.0.tar --name kwok- +rm /workdir/cache/image-archive/registry.k8s.io/kwok/kwok/v0.6.0.tar +kubectl config view --minify=true --raw=true +cat </workdir/clusters//kubeconfig.yaml +EOF +kubectl cordon kwok--control-plane +podman exec kwok--control-plane chmod -R +r /etc/kubernetes/pki +# Add context kwok- to ~/.kube/config diff --git a/test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster_with_extra.txt b/test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster_with_extra.txt new file mode 100644 index 000000000..6d98e13bf --- /dev/null +++ b/test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster_with_extra.txt @@ -0,0 +1,341 @@ +# Save cluster config to /workdir/clusters//kwok.yaml +mkdir -p /workdir/clusters/ +mkdir -p /workdir/clusters//pki +# Generate PKI to /workdir/clusters//pki +mkdir -p /workdir/clusters//pki/etcd +podman pull docker.io/kindest/node:v1.30.2 +mkdir -p /workdir/clusters//pki +mkdir -p /workdir/clusters//manifests +cat </workdir/clusters//kind.yaml +apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +kubeadmConfigPatches: +- | + apiServer: + extraArgs: + enable-priority-and-fairness: "false" + max-mutating-requests-inflight: "0" + max-requests-inflight: "0" + v: "5" + extraVolumes: + - hostPath: /var/components/apiserver/extras/tmp + mountPath: /extras/tmp + name: tmp-apiserver + pathType: DirectoryOrCreate + apiVersion: kubeadm.k8s.io/v1beta3 + controllerManager: + extraArgs: + kube-api-burst: "10000" + kube-api-qps: "5000" + v: "5" + extraVolumes: + - hostPath: /var/components/controller-manager/extras/tmp + mountPath: /extras/tmp + name: tmp-controller-manager + pathType: DirectoryOrCreate + dns: {} + etcd: + local: + dataDir: /var/lib/etcd + extraArgs: + log-level: debug + kind: ClusterConfiguration + networking: {} + scheduler: + extraArgs: + kube-api-burst: "10000" + kube-api-qps: "5000" + v: "5" + extraVolumes: + - hostPath: /var/components/scheduler/extras/tmp + mountPath: /extras/tmp + name: tmp-scheduler + pathType: DirectoryOrCreate +networking: + apiServerPort: 32766 +nodes: +- extraMounts: + - containerPath: /etc/kwok/ + hostPath: /workdir/clusters/ + - containerPath: /etc/kubernetes/manifests + hostPath: /workdir/clusters//manifests + - containerPath: /etc/kubernetes/pki + hostPath: /workdir/clusters//pki + - containerPath: /var/components/apiserver/extras/tmp + hostPath: /test/e2e/kwokctl/dryrun/extras/apiserver + - containerPath: /var/components/controller-manager/extras/tmp + hostPath: /test/e2e/kwokctl/dryrun/extras/controller-manager + - containerPath: /var/components/scheduler/extras/tmp + hostPath: /test/e2e/kwokctl/dryrun/extras/scheduler + - containerPath: /var/components/controller/extras/tmp + hostPath: /test/e2e/kwokctl/dryrun/extras/controller + - containerPath: /var/components/prometheus/extras/tmp + hostPath: /test/e2e/kwokctl/dryrun/extras/prometheus + - containerPath: /var/components/prometheus/etc/prometheus/prometheus.yaml + hostPath: /workdir/clusters//prometheus.yaml + extraPortMappings: + - containerPort: 9090 + hostPort: 9090 + protocol: TCP + - containerPort: 2379 + hostPort: 32765 + protocol: TCP + role: control-plane +EOF +podman pull registry.k8s.io/kwok/kwok:v0.6.0 +cat </workdir/clusters//manifests/kwok-controller.yaml +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: kwok-controller + namespace: kube-system +spec: + containers: + - args: + - --manage-all-nodes=false + - --manage-nodes-with-annotation-selector=kwok.x-k8s.io/node=fake + - --kubeconfig=/root/.kube/config + - --config=/root/.kwok/kwok.yaml + - --tls-cert-file=/etc/kubernetes/pki/admin.crt + - --tls-private-key-file=/etc/kubernetes/pki/admin.key + - --node-ip=$(POD_IP) + - --node-name=kwok-controller.kube-system.svc + - --node-port=10247 + - --server-address=0.0.0.0:10247 + - --node-lease-duration-seconds=40 + - --v=-4 + command: + - kwok + env: + - name: TEST_KEY + value: TEST_VALUE + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: registry.k8s.io/kwok/kwok:v0.6.0 + imagePullPolicy: Never + name: kwok-controller + resources: {} + volumeMounts: + - mountPath: /root/.kube/config + name: volume-0 + readOnly: true + - mountPath: /etc/kubernetes/pki/ca.crt + name: volume-1 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.crt + name: volume-2 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.key + name: volume-3 + readOnly: true + - mountPath: /root/.kwok/kwok.yaml + name: volume-4 + readOnly: true + - mountPath: /extras/tmp + name: tmp-controller + hostNetwork: true + restartPolicy: Always + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: volume-0 + - hostPath: + path: /etc/kubernetes/pki/ca.crt + name: volume-1 + - hostPath: + path: /etc/kubernetes/pki/admin.crt + name: volume-2 + - hostPath: + path: /etc/kubernetes/pki/admin.key + name: volume-3 + - hostPath: + path: /etc/kwok/kwok.yaml + name: volume-4 + - hostPath: + path: /test/e2e/kwokctl/dryrun/extras/controller + type: DirectoryOrCreate + name: tmp-controller +status: {} +EOF +podman pull docker.io/prom/prometheus:v2.53.0 +cat </workdir/clusters//manifests/prometheus.yaml +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: prometheus + namespace: kube-system +spec: + containers: + - args: + - --config.file=/etc/prometheus/prometheus.yaml + - --web.listen-address=0.0.0.0:9090 + - --log.level=debug + command: + - prometheus + env: + - name: TEST_KEY + value: TEST_VALUE + image: docker.io/prom/prometheus:v2.53.0 + imagePullPolicy: Never + name: prometheus + ports: + - containerPort: 9090 + hostPort: 9090 + resources: {} + volumeMounts: + - mountPath: /etc/prometheus/prometheus.yaml + name: volume-0 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.crt + name: volume-1 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.key + name: volume-2 + readOnly: true + - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.crt + name: volume-3 + readOnly: true + - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.key + name: volume-4 + readOnly: true + - mountPath: /extras/tmp + name: tmp-prometheus + hostNetwork: true + restartPolicy: Always + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: + - hostPath: + path: /var/components/prometheus/etc/prometheus/prometheus.yaml + name: volume-0 + - hostPath: + path: /etc/kubernetes/pki/admin.crt + name: volume-1 + - hostPath: + path: /etc/kubernetes/pki/admin.key + name: volume-2 + - hostPath: + path: /etc/kubernetes/pki/apiserver-etcd-client.crt + name: volume-3 + - hostPath: + path: /etc/kubernetes/pki/apiserver-etcd-client.key + name: volume-4 + - hostPath: + path: /test/e2e/kwokctl/dryrun/extras/prometheus + type: DirectoryOrCreate + name: tmp-prometheus +status: {} +EOF +cat </workdir/clusters//prometheus.yaml +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - follow_redirects: true + enable_http2: true + scheme: http + timeout: 10s + api_version: v2 + static_configs: + - targets: [] +scrape_configs: +- job_name: "etcd" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/apiserver-etcd-client.crt" + key_file: "/etc/kubernetes/pki/apiserver-etcd-client.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:2379 +- job_name: "kube-apiserver" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:6443 +- job_name: "kube-controller-manager" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10257 +- job_name: "kube-scheduler" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10259 +- job_name: "kwok-controller-metrics-discovery" + http_sd_configs: + - url: http://127.0.0.1:10247/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:10247 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 +EOF +# Save cluster config to /workdir/clusters//kwok.yaml +KIND_EXPERIMENTAL_PROVIDER=podman kind create cluster --config /workdir/clusters//kind.yaml --name kwok- --image docker.io/kindest/node:v1.30.2 --wait 29m +mkdir -p /workdir/cache/image-archive/registry.k8s.io/kwok/kwok +podman save registry.k8s.io/kwok/kwok:v0.6.0 -o /workdir/cache/image-archive/registry.k8s.io/kwok/kwok/v0.6.0.tar +KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/registry.k8s.io/kwok/kwok/v0.6.0.tar --name kwok- +rm /workdir/cache/image-archive/registry.k8s.io/kwok/kwok/v0.6.0.tar +mkdir -p /workdir/cache/image-archive/docker.io/prom/prometheus +podman save docker.io/prom/prometheus:v2.53.0 -o /workdir/cache/image-archive/docker.io/prom/prometheus/v2.53.0.tar +KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/docker.io/prom/prometheus/v2.53.0.tar --name kwok- +rm /workdir/cache/image-archive/docker.io/prom/prometheus/v2.53.0.tar +kubectl config view --minify=true --raw=true +cat </workdir/clusters//kubeconfig.yaml +EOF +kubectl cordon kwok--control-plane +podman exec kwok--control-plane chmod -R +r /etc/kubernetes/pki +# Add context kwok- to ~/.kube/config diff --git a/test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster_with_verbosity.txt b/test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster_with_verbosity.txt new file mode 100644 index 000000000..92cf56424 --- /dev/null +++ b/test/e2e/kwokctl/dryrun/testdata/kind-podman/create_cluster_with_verbosity.txt @@ -0,0 +1,582 @@ +# Save cluster config to /workdir/clusters//kwok.yaml +mkdir -p /workdir/clusters/ +mkdir -p /workdir/clusters//pki +# Generate PKI to /workdir/clusters//pki +mkdir -p /workdir/clusters//pki/etcd +podman pull docker.io/kindest/node:v1.30.2 +mkdir -p /workdir/clusters//pki +mkdir -p /workdir/clusters//manifests +mkdir -p /workdir/clusters//logs +touch /workdir/clusters//logs/audit.log +cp /test/kwokctl/audit-policy.yaml /workdir/clusters//audit.yaml +cp /test/kwokctl/scheduler-config.yaml /workdir/clusters//scheduler.yaml +cat <>/workdir/clusters//scheduler.yaml +clientConnection: + kubeconfig: "/etc/kubernetes/scheduler.conf" +EOF +cat </workdir/clusters//apiserver-tracing-config.yaml +apiVersion: apiserver.config.k8s.io/v1alpha1 +kind: TracingConfiguration +endpoint: 0.0.0.0:4317 +samplingRatePerMillion: 1000000 +EOF +cat </workdir/clusters//kind.yaml +apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +kubeadmConfigPatches: +- | + apiServer: + extraArgs: + audit-log-path: /var/log/kubernetes/audit.log + audit-policy-file: /etc/kubernetes/audit/audit.yaml + enable-priority-and-fairness: "false" + max-mutating-requests-inflight: "0" + max-requests-inflight: "0" + tracing-config-file: /etc/kubernetes/apiserver-tracing-config.yaml + extraVolumes: + - hostPath: /var/components/apiserver/etc/kubernetes/audit/audit.yaml + mountPath: /etc/kubernetes/audit/audit.yaml + name: audit-policy-file + pathType: File + readOnly: true + - hostPath: /var/components/apiserver/var/log/kubernetes/audit.log + mountPath: /var/log/kubernetes/audit.log + name: audit-log-path + pathType: File + - hostPath: /var/components/apiserver/etc/kubernetes/apiserver-tracing-config.yaml + mountPath: /etc/kubernetes/apiserver-tracing-config.yaml + name: apiserver-tracing-config + pathType: File + readOnly: true + apiVersion: kubeadm.k8s.io/v1beta3 + controllerManager: + extraArgs: + kube-api-burst: "10000" + kube-api-qps: "5000" + dns: {} + etcd: + local: + dataDir: /var/lib/etcd + kind: ClusterConfiguration + networking: {} + scheduler: + extraArgs: + config: /etc/kubernetes/scheduler/scheduler.yaml + kube-api-burst: "10000" + kube-api-qps: "5000" + extraVolumes: + - hostPath: /var/components/scheduler/etc/kubernetes/scheduler/scheduler.yaml + mountPath: /etc/kubernetes/scheduler/scheduler.yaml + name: config + pathType: File + readOnly: true +networking: + apiServerPort: 32766 +nodes: +- extraMounts: + - containerPath: /etc/kwok/ + hostPath: /workdir/clusters/ + - containerPath: /etc/kubernetes/manifests + hostPath: /workdir/clusters//manifests + - containerPath: /etc/kubernetes/pki + hostPath: /workdir/clusters//pki + - containerPath: /var/components/apiserver/etc/kubernetes/audit/audit.yaml + hostPath: /workdir/clusters//audit.yaml + readOnly: true + - containerPath: /var/components/apiserver/var/log/kubernetes/audit.log + hostPath: /workdir/clusters//logs/audit.log + - containerPath: /var/components/apiserver/etc/kubernetes/apiserver-tracing-config.yaml + hostPath: /workdir/clusters//apiserver-tracing-config.yaml + readOnly: true + - containerPath: /var/components/scheduler/etc/kubernetes/scheduler/scheduler.yaml + hostPath: /workdir/clusters//scheduler.yaml + readOnly: true + - containerPath: /var/components/prometheus/etc/prometheus/prometheus.yaml + hostPath: /workdir/clusters//prometheus.yaml + extraPortMappings: + - containerPort: 8080 + hostPort: 8000 + protocol: TCP + - containerPort: 9090 + hostPort: 9090 + protocol: TCP + - containerPort: 16686 + hostPort: 16686 + protocol: TCP + - containerPort: 2379 + hostPort: 32765 + protocol: TCP + role: control-plane +EOF +podman pull registry.k8s.io/kwok/kwok:v0.6.0 +cat </workdir/clusters//manifests/kwok-controller.yaml +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: kwok-controller + namespace: kube-system +spec: + containers: + - args: + - --manage-all-nodes=false + - --manage-nodes-with-annotation-selector=kwok.x-k8s.io/node=fake + - --kubeconfig=/root/.kube/config + - --config=/root/.kwok/kwok.yaml + - --tls-cert-file=/etc/kubernetes/pki/admin.crt + - --tls-private-key-file=/etc/kubernetes/pki/admin.key + - --node-ip=$(POD_IP) + - --node-name=kwok-controller.kube-system.svc + - --node-port=10247 + - --server-address=0.0.0.0:10247 + - --node-lease-duration-seconds=40 + command: + - kwok + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: registry.k8s.io/kwok/kwok:v0.6.0 + imagePullPolicy: Never + name: kwok-controller + resources: {} + volumeMounts: + - mountPath: /root/.kube/config + name: volume-0 + readOnly: true + - mountPath: /etc/kubernetes/pki/ca.crt + name: volume-1 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.crt + name: volume-2 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.key + name: volume-3 + readOnly: true + - mountPath: /root/.kwok/kwok.yaml + name: volume-4 + readOnly: true + hostNetwork: true + restartPolicy: Always + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: volume-0 + - hostPath: + path: /etc/kubernetes/pki/ca.crt + name: volume-1 + - hostPath: + path: /etc/kubernetes/pki/admin.crt + name: volume-2 + - hostPath: + path: /etc/kubernetes/pki/admin.key + name: volume-3 + - hostPath: + path: /etc/kwok/kwok.yaml + name: volume-4 +status: {} +EOF +podman pull docker.io/kubernetesui/dashboard:v2.7.0 +cat </workdir/clusters//manifests/dashboard.yaml +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: dashboard + namespace: kube-system +spec: + containers: + - args: + - --insecure-bind-address=0.0.0.0 + - --bind-address=127.0.0.1 + - --port=0 + - --enable-insecure-login + - --enable-skip-login + - --disable-settings-authorizer + - --sidecar-host=127.0.0.1:8000 + - --system-banner=Welcome to kwok- + - --kubeconfig=/root/.kube/config + - --insecure-port=8080 + image: docker.io/kubernetesui/dashboard:v2.7.0 + imagePullPolicy: Never + name: dashboard + ports: + - containerPort: 8080 + hostPort: 8080 + name: http + protocol: TCP + resources: {} + volumeMounts: + - mountPath: /root/.kube/config + name: volume-0 + readOnly: true + - mountPath: /etc/kubernetes/pki/ca.crt + name: volume-1 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.crt + name: volume-2 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.key + name: volume-3 + readOnly: true + hostNetwork: true + restartPolicy: Always + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: volume-0 + - hostPath: + path: /etc/kubernetes/pki/ca.crt + name: volume-1 + - hostPath: + path: /etc/kubernetes/pki/admin.crt + name: volume-2 + - hostPath: + path: /etc/kubernetes/pki/admin.key + name: volume-3 +status: {} +EOF +podman pull docker.io/kubernetesui/metrics-scraper:v1.0.9 +cat </workdir/clusters//manifests/dashboard-metrics-scraper.yaml +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: dashboard-metrics-scraper + namespace: kube-system +spec: + containers: + - args: + - --db-file=/metrics.db + - --kubeconfig=/root/.kube/config + image: docker.io/kubernetesui/metrics-scraper:v1.0.9 + imagePullPolicy: Never + name: dashboard-metrics-scraper + resources: {} + volumeMounts: + - mountPath: /root/.kube/config + name: volume-0 + readOnly: true + - mountPath: /etc/kubernetes/pki/ca.crt + name: volume-1 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.crt + name: volume-2 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.key + name: volume-3 + readOnly: true + hostNetwork: true + restartPolicy: Always + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: volume-0 + - hostPath: + path: /etc/kubernetes/pki/ca.crt + name: volume-1 + - hostPath: + path: /etc/kubernetes/pki/admin.crt + name: volume-2 + - hostPath: + path: /etc/kubernetes/pki/admin.key + name: volume-3 +status: {} +EOF +podman pull registry.k8s.io/metrics-server/metrics-server:v0.7.1 +cat </workdir/clusters//manifests/metrics-server.yaml +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: metrics-server + namespace: kube-system +spec: + containers: + - args: + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-use-node-status-port + - --kubelet-insecure-tls + - --metric-resolution=15s + - --bind-address=0.0.0.0 + - --secure-port=4443 + - --kubeconfig=/root/.kube/config + - --authentication-kubeconfig=/root/.kube/config + - --authorization-kubeconfig=/root/.kube/config + - --tls-cert-file=/etc/kubernetes/pki/admin.crt + - --tls-private-key-file=/etc/kubernetes/pki/admin.key + command: + - /metrics-server + image: registry.k8s.io/metrics-server/metrics-server:v0.7.1 + imagePullPolicy: Never + name: metrics-server + ports: + - containerPort: 4443 + hostPort: 4443 + resources: {} + volumeMounts: + - mountPath: /root/.kube/config + name: volume-0 + readOnly: true + - mountPath: /etc/kubernetes/pki/ca.crt + name: volume-1 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.crt + name: volume-2 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.key + name: volume-3 + readOnly: true + hostNetwork: true + restartPolicy: Always + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: volume-0 + - hostPath: + path: /etc/kubernetes/pki/ca.crt + name: volume-1 + - hostPath: + path: /etc/kubernetes/pki/admin.crt + name: volume-2 + - hostPath: + path: /etc/kubernetes/pki/admin.key + name: volume-3 +status: {} +EOF +podman pull docker.io/prom/prometheus:v2.53.0 +cat </workdir/clusters//manifests/prometheus.yaml +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: prometheus + namespace: kube-system +spec: + containers: + - args: + - --config.file=/etc/prometheus/prometheus.yaml + - --web.listen-address=0.0.0.0:9090 + command: + - prometheus + image: docker.io/prom/prometheus:v2.53.0 + imagePullPolicy: Never + name: prometheus + ports: + - containerPort: 9090 + hostPort: 9090 + resources: {} + volumeMounts: + - mountPath: /etc/prometheus/prometheus.yaml + name: volume-0 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.crt + name: volume-1 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.key + name: volume-2 + readOnly: true + - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.crt + name: volume-3 + readOnly: true + - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.key + name: volume-4 + readOnly: true + hostNetwork: true + restartPolicy: Always + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: + - hostPath: + path: /var/components/prometheus/etc/prometheus/prometheus.yaml + name: volume-0 + - hostPath: + path: /etc/kubernetes/pki/admin.crt + name: volume-1 + - hostPath: + path: /etc/kubernetes/pki/admin.key + name: volume-2 + - hostPath: + path: /etc/kubernetes/pki/apiserver-etcd-client.crt + name: volume-3 + - hostPath: + path: /etc/kubernetes/pki/apiserver-etcd-client.key + name: volume-4 +status: {} +EOF +podman pull docker.io/jaegertracing/all-in-one:1.58.1 +cat </workdir/clusters//manifests/jaeger.yaml +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: jaeger + namespace: kube-system +spec: + containers: + - args: + - --collector.otlp.enabled=true + - --query.http-server.host-port=0.0.0.0:16686 + image: docker.io/jaegertracing/all-in-one:1.58.1 + imagePullPolicy: Never + name: jaeger + ports: + - containerPort: 16686 + hostPort: 16686 + resources: {} + hostNetwork: true + restartPolicy: Always + securityContext: + runAsGroup: 0 + runAsUser: 0 +status: {} +EOF +cat </workdir/clusters//prometheus.yaml +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - follow_redirects: true + enable_http2: true + scheme: http + timeout: 10s + api_version: v2 + static_configs: + - targets: [] +scrape_configs: +- job_name: "etcd" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/apiserver-etcd-client.crt" + key_file: "/etc/kubernetes/pki/apiserver-etcd-client.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:2379 +- job_name: "kube-apiserver" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:6443 +- job_name: "kube-controller-manager" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10257 +- job_name: "kube-scheduler" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10259 +- job_name: "kwok-controller-metrics-discovery" + http_sd_configs: + - url: http://127.0.0.1:10247/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:10247 +- job_name: "metrics-server" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:4443 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 +EOF +# Save cluster config to /workdir/clusters//kwok.yaml +KIND_EXPERIMENTAL_PROVIDER=podman kind create cluster --config /workdir/clusters//kind.yaml --name kwok- --image docker.io/kindest/node:v1.30.2 --wait 29m +mkdir -p /workdir/cache/image-archive/registry.k8s.io/kwok/kwok +podman save registry.k8s.io/kwok/kwok:v0.6.0 -o /workdir/cache/image-archive/registry.k8s.io/kwok/kwok/v0.6.0.tar +KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/registry.k8s.io/kwok/kwok/v0.6.0.tar --name kwok- +rm /workdir/cache/image-archive/registry.k8s.io/kwok/kwok/v0.6.0.tar +mkdir -p /workdir/cache/image-archive/docker.io/kubernetesui/dashboard +podman save docker.io/kubernetesui/dashboard:v2.7.0 -o /workdir/cache/image-archive/docker.io/kubernetesui/dashboard/v2.7.0.tar +KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/docker.io/kubernetesui/dashboard/v2.7.0.tar --name kwok- +rm /workdir/cache/image-archive/docker.io/kubernetesui/dashboard/v2.7.0.tar +mkdir -p /workdir/cache/image-archive/docker.io/kubernetesui/metrics-scraper +podman save docker.io/kubernetesui/metrics-scraper:v1.0.9 -o /workdir/cache/image-archive/docker.io/kubernetesui/metrics-scraper/v1.0.9.tar +KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/docker.io/kubernetesui/metrics-scraper/v1.0.9.tar --name kwok- +rm /workdir/cache/image-archive/docker.io/kubernetesui/metrics-scraper/v1.0.9.tar +mkdir -p /workdir/cache/image-archive/registry.k8s.io/metrics-server/metrics-server +podman save registry.k8s.io/metrics-server/metrics-server:v0.7.1 -o /workdir/cache/image-archive/registry.k8s.io/metrics-server/metrics-server/v0.7.1.tar +KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/registry.k8s.io/metrics-server/metrics-server/v0.7.1.tar --name kwok- +rm /workdir/cache/image-archive/registry.k8s.io/metrics-server/metrics-server/v0.7.1.tar +mkdir -p /workdir/cache/image-archive/docker.io/prom/prometheus +podman save docker.io/prom/prometheus:v2.53.0 -o /workdir/cache/image-archive/docker.io/prom/prometheus/v2.53.0.tar +KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/docker.io/prom/prometheus/v2.53.0.tar --name kwok- +rm /workdir/cache/image-archive/docker.io/prom/prometheus/v2.53.0.tar +mkdir -p /workdir/cache/image-archive/docker.io/jaegertracing/all-in-one +podman save docker.io/jaegertracing/all-in-one:1.58.1 -o /workdir/cache/image-archive/docker.io/jaegertracing/all-in-one/1.58.1.tar +KIND_EXPERIMENTAL_PROVIDER=podman kind load image-archive /workdir/cache/image-archive/docker.io/jaegertracing/all-in-one/1.58.1.tar --name kwok- +rm /workdir/cache/image-archive/docker.io/jaegertracing/all-in-one/1.58.1.tar +kubectl config view --minify=true --raw=true +cat </workdir/clusters//kubeconfig.yaml +EOF +kubectl cordon kwok--control-plane +podman exec kwok--control-plane chmod -R +r /etc/kubernetes/pki +# Set up apiservice for metrics server +# Add context kwok- to ~/.kube/config diff --git a/test/e2e/kwokctl/dryrun/testdata/kind/create_cluster.txt b/test/e2e/kwokctl/dryrun/testdata/kind/create_cluster.txt new file mode 100644 index 000000000..85d153aa1 --- /dev/null +++ b/test/e2e/kwokctl/dryrun/testdata/kind/create_cluster.txt @@ -0,0 +1,130 @@ +# Save cluster config to /workdir/clusters//kwok.yaml +mkdir -p /workdir/clusters/ +mkdir -p /workdir/clusters//pki +# Generate PKI to /workdir/clusters//pki +mkdir -p /workdir/clusters//pki/etcd +docker pull docker.io/kindest/node:v1.30.2 +mkdir -p /workdir/clusters//pki +mkdir -p /workdir/clusters//manifests +cat </workdir/clusters//kind.yaml +apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +kubeadmConfigPatches: +- | + apiServer: + extraArgs: + enable-priority-and-fairness: "false" + max-mutating-requests-inflight: "0" + max-requests-inflight: "0" + apiVersion: kubeadm.k8s.io/v1beta3 + controllerManager: + extraArgs: + kube-api-burst: "10000" + kube-api-qps: "5000" + dns: {} + etcd: + local: + dataDir: /var/lib/etcd + kind: ClusterConfiguration + networking: {} + scheduler: + extraArgs: + kube-api-burst: "10000" + kube-api-qps: "5000" +networking: + apiServerPort: 32766 +nodes: +- extraMounts: + - containerPath: /etc/kwok/ + hostPath: /workdir/clusters/ + - containerPath: /etc/kubernetes/manifests + hostPath: /workdir/clusters//manifests + - containerPath: /etc/kubernetes/pki + hostPath: /workdir/clusters//pki + extraPortMappings: + - containerPort: 2379 + hostPort: 32765 + protocol: TCP + role: control-plane +EOF +docker pull registry.k8s.io/kwok/kwok:v0.6.0 +cat </workdir/clusters//manifests/kwok-controller.yaml +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: kwok-controller + namespace: kube-system +spec: + containers: + - args: + - --manage-all-nodes=false + - --manage-nodes-with-annotation-selector=kwok.x-k8s.io/node=fake + - --kubeconfig=/root/.kube/config + - --config=/root/.kwok/kwok.yaml + - --tls-cert-file=/etc/kubernetes/pki/admin.crt + - --tls-private-key-file=/etc/kubernetes/pki/admin.key + - --node-ip=$(POD_IP) + - --node-name=kwok-controller.kube-system.svc + - --node-port=10247 + - --server-address=0.0.0.0:10247 + - --node-lease-duration-seconds=40 + command: + - kwok + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: registry.k8s.io/kwok/kwok:v0.6.0 + imagePullPolicy: Never + name: kwok-controller + resources: {} + volumeMounts: + - mountPath: /root/.kube/config + name: volume-0 + readOnly: true + - mountPath: /etc/kubernetes/pki/ca.crt + name: volume-1 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.crt + name: volume-2 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.key + name: volume-3 + readOnly: true + - mountPath: /root/.kwok/kwok.yaml + name: volume-4 + readOnly: true + hostNetwork: true + restartPolicy: Always + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: volume-0 + - hostPath: + path: /etc/kubernetes/pki/ca.crt + name: volume-1 + - hostPath: + path: /etc/kubernetes/pki/admin.crt + name: volume-2 + - hostPath: + path: /etc/kubernetes/pki/admin.key + name: volume-3 + - hostPath: + path: /etc/kwok/kwok.yaml + name: volume-4 +status: {} +EOF +# Save cluster config to /workdir/clusters//kwok.yaml +KIND_EXPERIMENTAL_PROVIDER=docker kind create cluster --config /workdir/clusters//kind.yaml --name kwok- --image docker.io/kindest/node:v1.30.2 --wait 29m +KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image registry.k8s.io/kwok/kwok:v0.6.0 --name kwok- +kubectl config view --minify=true --raw=true +cat </workdir/clusters//kubeconfig.yaml +EOF +kubectl cordon kwok--control-plane +docker exec kwok--control-plane chmod -R +r /etc/kubernetes/pki +# Add context kwok- to ~/.kube/config diff --git a/test/e2e/kwokctl/dryrun/testdata/kind/create_cluster_with_extra.txt b/test/e2e/kwokctl/dryrun/testdata/kind/create_cluster_with_extra.txt new file mode 100644 index 000000000..8b78c33f5 --- /dev/null +++ b/test/e2e/kwokctl/dryrun/testdata/kind/create_cluster_with_extra.txt @@ -0,0 +1,335 @@ +# Save cluster config to /workdir/clusters//kwok.yaml +mkdir -p /workdir/clusters/ +mkdir -p /workdir/clusters//pki +# Generate PKI to /workdir/clusters//pki +mkdir -p /workdir/clusters//pki/etcd +docker pull docker.io/kindest/node:v1.30.2 +mkdir -p /workdir/clusters//pki +mkdir -p /workdir/clusters//manifests +cat </workdir/clusters//kind.yaml +apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +kubeadmConfigPatches: +- | + apiServer: + extraArgs: + enable-priority-and-fairness: "false" + max-mutating-requests-inflight: "0" + max-requests-inflight: "0" + v: "5" + extraVolumes: + - hostPath: /var/components/apiserver/extras/tmp + mountPath: /extras/tmp + name: tmp-apiserver + pathType: DirectoryOrCreate + apiVersion: kubeadm.k8s.io/v1beta3 + controllerManager: + extraArgs: + kube-api-burst: "10000" + kube-api-qps: "5000" + v: "5" + extraVolumes: + - hostPath: /var/components/controller-manager/extras/tmp + mountPath: /extras/tmp + name: tmp-controller-manager + pathType: DirectoryOrCreate + dns: {} + etcd: + local: + dataDir: /var/lib/etcd + extraArgs: + log-level: debug + kind: ClusterConfiguration + networking: {} + scheduler: + extraArgs: + kube-api-burst: "10000" + kube-api-qps: "5000" + v: "5" + extraVolumes: + - hostPath: /var/components/scheduler/extras/tmp + mountPath: /extras/tmp + name: tmp-scheduler + pathType: DirectoryOrCreate +networking: + apiServerPort: 32766 +nodes: +- extraMounts: + - containerPath: /etc/kwok/ + hostPath: /workdir/clusters/ + - containerPath: /etc/kubernetes/manifests + hostPath: /workdir/clusters//manifests + - containerPath: /etc/kubernetes/pki + hostPath: /workdir/clusters//pki + - containerPath: /var/components/apiserver/extras/tmp + hostPath: /test/e2e/kwokctl/dryrun/extras/apiserver + - containerPath: /var/components/controller-manager/extras/tmp + hostPath: /test/e2e/kwokctl/dryrun/extras/controller-manager + - containerPath: /var/components/scheduler/extras/tmp + hostPath: /test/e2e/kwokctl/dryrun/extras/scheduler + - containerPath: /var/components/controller/extras/tmp + hostPath: /test/e2e/kwokctl/dryrun/extras/controller + - containerPath: /var/components/prometheus/extras/tmp + hostPath: /test/e2e/kwokctl/dryrun/extras/prometheus + - containerPath: /var/components/prometheus/etc/prometheus/prometheus.yaml + hostPath: /workdir/clusters//prometheus.yaml + extraPortMappings: + - containerPort: 9090 + hostPort: 9090 + protocol: TCP + - containerPort: 2379 + hostPort: 32765 + protocol: TCP + role: control-plane +EOF +docker pull registry.k8s.io/kwok/kwok:v0.6.0 +cat </workdir/clusters//manifests/kwok-controller.yaml +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: kwok-controller + namespace: kube-system +spec: + containers: + - args: + - --manage-all-nodes=false + - --manage-nodes-with-annotation-selector=kwok.x-k8s.io/node=fake + - --kubeconfig=/root/.kube/config + - --config=/root/.kwok/kwok.yaml + - --tls-cert-file=/etc/kubernetes/pki/admin.crt + - --tls-private-key-file=/etc/kubernetes/pki/admin.key + - --node-ip=$(POD_IP) + - --node-name=kwok-controller.kube-system.svc + - --node-port=10247 + - --server-address=0.0.0.0:10247 + - --node-lease-duration-seconds=40 + - --v=-4 + command: + - kwok + env: + - name: TEST_KEY + value: TEST_VALUE + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: registry.k8s.io/kwok/kwok:v0.6.0 + imagePullPolicy: Never + name: kwok-controller + resources: {} + volumeMounts: + - mountPath: /root/.kube/config + name: volume-0 + readOnly: true + - mountPath: /etc/kubernetes/pki/ca.crt + name: volume-1 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.crt + name: volume-2 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.key + name: volume-3 + readOnly: true + - mountPath: /root/.kwok/kwok.yaml + name: volume-4 + readOnly: true + - mountPath: /extras/tmp + name: tmp-controller + hostNetwork: true + restartPolicy: Always + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: volume-0 + - hostPath: + path: /etc/kubernetes/pki/ca.crt + name: volume-1 + - hostPath: + path: /etc/kubernetes/pki/admin.crt + name: volume-2 + - hostPath: + path: /etc/kubernetes/pki/admin.key + name: volume-3 + - hostPath: + path: /etc/kwok/kwok.yaml + name: volume-4 + - hostPath: + path: /test/e2e/kwokctl/dryrun/extras/controller + type: DirectoryOrCreate + name: tmp-controller +status: {} +EOF +docker pull docker.io/prom/prometheus:v2.53.0 +cat </workdir/clusters//manifests/prometheus.yaml +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: prometheus + namespace: kube-system +spec: + containers: + - args: + - --config.file=/etc/prometheus/prometheus.yaml + - --web.listen-address=0.0.0.0:9090 + - --log.level=debug + command: + - prometheus + env: + - name: TEST_KEY + value: TEST_VALUE + image: docker.io/prom/prometheus:v2.53.0 + imagePullPolicy: Never + name: prometheus + ports: + - containerPort: 9090 + hostPort: 9090 + resources: {} + volumeMounts: + - mountPath: /etc/prometheus/prometheus.yaml + name: volume-0 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.crt + name: volume-1 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.key + name: volume-2 + readOnly: true + - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.crt + name: volume-3 + readOnly: true + - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.key + name: volume-4 + readOnly: true + - mountPath: /extras/tmp + name: tmp-prometheus + hostNetwork: true + restartPolicy: Always + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: + - hostPath: + path: /var/components/prometheus/etc/prometheus/prometheus.yaml + name: volume-0 + - hostPath: + path: /etc/kubernetes/pki/admin.crt + name: volume-1 + - hostPath: + path: /etc/kubernetes/pki/admin.key + name: volume-2 + - hostPath: + path: /etc/kubernetes/pki/apiserver-etcd-client.crt + name: volume-3 + - hostPath: + path: /etc/kubernetes/pki/apiserver-etcd-client.key + name: volume-4 + - hostPath: + path: /test/e2e/kwokctl/dryrun/extras/prometheus + type: DirectoryOrCreate + name: tmp-prometheus +status: {} +EOF +cat </workdir/clusters//prometheus.yaml +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - follow_redirects: true + enable_http2: true + scheme: http + timeout: 10s + api_version: v2 + static_configs: + - targets: [] +scrape_configs: +- job_name: "etcd" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/apiserver-etcd-client.crt" + key_file: "/etc/kubernetes/pki/apiserver-etcd-client.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:2379 +- job_name: "kube-apiserver" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:6443 +- job_name: "kube-controller-manager" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10257 +- job_name: "kube-scheduler" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10259 +- job_name: "kwok-controller-metrics-discovery" + http_sd_configs: + - url: http://127.0.0.1:10247/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:10247 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 +EOF +# Save cluster config to /workdir/clusters//kwok.yaml +KIND_EXPERIMENTAL_PROVIDER=docker kind create cluster --config /workdir/clusters//kind.yaml --name kwok- --image docker.io/kindest/node:v1.30.2 --wait 29m +KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image registry.k8s.io/kwok/kwok:v0.6.0 --name kwok- +KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image docker.io/prom/prometheus:v2.53.0 --name kwok- +kubectl config view --minify=true --raw=true +cat </workdir/clusters//kubeconfig.yaml +EOF +kubectl cordon kwok--control-plane +docker exec kwok--control-plane chmod -R +r /etc/kubernetes/pki +# Add context kwok- to ~/.kube/config diff --git a/test/e2e/kwokctl/dryrun/testdata/kind/create_cluster_with_verbosity.txt b/test/e2e/kwokctl/dryrun/testdata/kind/create_cluster_with_verbosity.txt new file mode 100644 index 000000000..3608c72d6 --- /dev/null +++ b/test/e2e/kwokctl/dryrun/testdata/kind/create_cluster_with_verbosity.txt @@ -0,0 +1,564 @@ +# Save cluster config to /workdir/clusters//kwok.yaml +mkdir -p /workdir/clusters/ +mkdir -p /workdir/clusters//pki +# Generate PKI to /workdir/clusters//pki +mkdir -p /workdir/clusters//pki/etcd +docker pull docker.io/kindest/node:v1.30.2 +mkdir -p /workdir/clusters//pki +mkdir -p /workdir/clusters//manifests +mkdir -p /workdir/clusters//logs +touch /workdir/clusters//logs/audit.log +cp /test/kwokctl/audit-policy.yaml /workdir/clusters//audit.yaml +cp /test/kwokctl/scheduler-config.yaml /workdir/clusters//scheduler.yaml +cat <>/workdir/clusters//scheduler.yaml +clientConnection: + kubeconfig: "/etc/kubernetes/scheduler.conf" +EOF +cat </workdir/clusters//apiserver-tracing-config.yaml +apiVersion: apiserver.config.k8s.io/v1alpha1 +kind: TracingConfiguration +endpoint: 0.0.0.0:4317 +samplingRatePerMillion: 1000000 +EOF +cat </workdir/clusters//kind.yaml +apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +kubeadmConfigPatches: +- | + apiServer: + extraArgs: + audit-log-path: /var/log/kubernetes/audit.log + audit-policy-file: /etc/kubernetes/audit/audit.yaml + enable-priority-and-fairness: "false" + max-mutating-requests-inflight: "0" + max-requests-inflight: "0" + tracing-config-file: /etc/kubernetes/apiserver-tracing-config.yaml + extraVolumes: + - hostPath: /var/components/apiserver/etc/kubernetes/audit/audit.yaml + mountPath: /etc/kubernetes/audit/audit.yaml + name: audit-policy-file + pathType: File + readOnly: true + - hostPath: /var/components/apiserver/var/log/kubernetes/audit.log + mountPath: /var/log/kubernetes/audit.log + name: audit-log-path + pathType: File + - hostPath: /var/components/apiserver/etc/kubernetes/apiserver-tracing-config.yaml + mountPath: /etc/kubernetes/apiserver-tracing-config.yaml + name: apiserver-tracing-config + pathType: File + readOnly: true + apiVersion: kubeadm.k8s.io/v1beta3 + controllerManager: + extraArgs: + kube-api-burst: "10000" + kube-api-qps: "5000" + dns: {} + etcd: + local: + dataDir: /var/lib/etcd + kind: ClusterConfiguration + networking: {} + scheduler: + extraArgs: + config: /etc/kubernetes/scheduler/scheduler.yaml + kube-api-burst: "10000" + kube-api-qps: "5000" + extraVolumes: + - hostPath: /var/components/scheduler/etc/kubernetes/scheduler/scheduler.yaml + mountPath: /etc/kubernetes/scheduler/scheduler.yaml + name: config + pathType: File + readOnly: true +networking: + apiServerPort: 32766 +nodes: +- extraMounts: + - containerPath: /etc/kwok/ + hostPath: /workdir/clusters/ + - containerPath: /etc/kubernetes/manifests + hostPath: /workdir/clusters//manifests + - containerPath: /etc/kubernetes/pki + hostPath: /workdir/clusters//pki + - containerPath: /var/components/apiserver/etc/kubernetes/audit/audit.yaml + hostPath: /workdir/clusters//audit.yaml + readOnly: true + - containerPath: /var/components/apiserver/var/log/kubernetes/audit.log + hostPath: /workdir/clusters//logs/audit.log + - containerPath: /var/components/apiserver/etc/kubernetes/apiserver-tracing-config.yaml + hostPath: /workdir/clusters//apiserver-tracing-config.yaml + readOnly: true + - containerPath: /var/components/scheduler/etc/kubernetes/scheduler/scheduler.yaml + hostPath: /workdir/clusters//scheduler.yaml + readOnly: true + - containerPath: /var/components/prometheus/etc/prometheus/prometheus.yaml + hostPath: /workdir/clusters//prometheus.yaml + extraPortMappings: + - containerPort: 8080 + hostPort: 8000 + protocol: TCP + - containerPort: 9090 + hostPort: 9090 + protocol: TCP + - containerPort: 16686 + hostPort: 16686 + protocol: TCP + - containerPort: 2379 + hostPort: 32765 + protocol: TCP + role: control-plane +EOF +docker pull registry.k8s.io/kwok/kwok:v0.6.0 +cat </workdir/clusters//manifests/kwok-controller.yaml +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: kwok-controller + namespace: kube-system +spec: + containers: + - args: + - --manage-all-nodes=false + - --manage-nodes-with-annotation-selector=kwok.x-k8s.io/node=fake + - --kubeconfig=/root/.kube/config + - --config=/root/.kwok/kwok.yaml + - --tls-cert-file=/etc/kubernetes/pki/admin.crt + - --tls-private-key-file=/etc/kubernetes/pki/admin.key + - --node-ip=$(POD_IP) + - --node-name=kwok-controller.kube-system.svc + - --node-port=10247 + - --server-address=0.0.0.0:10247 + - --node-lease-duration-seconds=40 + command: + - kwok + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: registry.k8s.io/kwok/kwok:v0.6.0 + imagePullPolicy: Never + name: kwok-controller + resources: {} + volumeMounts: + - mountPath: /root/.kube/config + name: volume-0 + readOnly: true + - mountPath: /etc/kubernetes/pki/ca.crt + name: volume-1 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.crt + name: volume-2 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.key + name: volume-3 + readOnly: true + - mountPath: /root/.kwok/kwok.yaml + name: volume-4 + readOnly: true + hostNetwork: true + restartPolicy: Always + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: volume-0 + - hostPath: + path: /etc/kubernetes/pki/ca.crt + name: volume-1 + - hostPath: + path: /etc/kubernetes/pki/admin.crt + name: volume-2 + - hostPath: + path: /etc/kubernetes/pki/admin.key + name: volume-3 + - hostPath: + path: /etc/kwok/kwok.yaml + name: volume-4 +status: {} +EOF +docker pull docker.io/kubernetesui/dashboard:v2.7.0 +cat </workdir/clusters//manifests/dashboard.yaml +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: dashboard + namespace: kube-system +spec: + containers: + - args: + - --insecure-bind-address=0.0.0.0 + - --bind-address=127.0.0.1 + - --port=0 + - --enable-insecure-login + - --enable-skip-login + - --disable-settings-authorizer + - --sidecar-host=127.0.0.1:8000 + - --system-banner=Welcome to kwok- + - --kubeconfig=/root/.kube/config + - --insecure-port=8080 + image: docker.io/kubernetesui/dashboard:v2.7.0 + imagePullPolicy: Never + name: dashboard + ports: + - containerPort: 8080 + hostPort: 8080 + name: http + protocol: TCP + resources: {} + volumeMounts: + - mountPath: /root/.kube/config + name: volume-0 + readOnly: true + - mountPath: /etc/kubernetes/pki/ca.crt + name: volume-1 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.crt + name: volume-2 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.key + name: volume-3 + readOnly: true + hostNetwork: true + restartPolicy: Always + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: volume-0 + - hostPath: + path: /etc/kubernetes/pki/ca.crt + name: volume-1 + - hostPath: + path: /etc/kubernetes/pki/admin.crt + name: volume-2 + - hostPath: + path: /etc/kubernetes/pki/admin.key + name: volume-3 +status: {} +EOF +docker pull docker.io/kubernetesui/metrics-scraper:v1.0.9 +cat </workdir/clusters//manifests/dashboard-metrics-scraper.yaml +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: dashboard-metrics-scraper + namespace: kube-system +spec: + containers: + - args: + - --db-file=/metrics.db + - --kubeconfig=/root/.kube/config + image: docker.io/kubernetesui/metrics-scraper:v1.0.9 + imagePullPolicy: Never + name: dashboard-metrics-scraper + resources: {} + volumeMounts: + - mountPath: /root/.kube/config + name: volume-0 + readOnly: true + - mountPath: /etc/kubernetes/pki/ca.crt + name: volume-1 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.crt + name: volume-2 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.key + name: volume-3 + readOnly: true + hostNetwork: true + restartPolicy: Always + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: volume-0 + - hostPath: + path: /etc/kubernetes/pki/ca.crt + name: volume-1 + - hostPath: + path: /etc/kubernetes/pki/admin.crt + name: volume-2 + - hostPath: + path: /etc/kubernetes/pki/admin.key + name: volume-3 +status: {} +EOF +docker pull registry.k8s.io/metrics-server/metrics-server:v0.7.1 +cat </workdir/clusters//manifests/metrics-server.yaml +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: metrics-server + namespace: kube-system +spec: + containers: + - args: + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname + - --kubelet-use-node-status-port + - --kubelet-insecure-tls + - --metric-resolution=15s + - --bind-address=0.0.0.0 + - --secure-port=4443 + - --kubeconfig=/root/.kube/config + - --authentication-kubeconfig=/root/.kube/config + - --authorization-kubeconfig=/root/.kube/config + - --tls-cert-file=/etc/kubernetes/pki/admin.crt + - --tls-private-key-file=/etc/kubernetes/pki/admin.key + command: + - /metrics-server + image: registry.k8s.io/metrics-server/metrics-server:v0.7.1 + imagePullPolicy: Never + name: metrics-server + ports: + - containerPort: 4443 + hostPort: 4443 + resources: {} + volumeMounts: + - mountPath: /root/.kube/config + name: volume-0 + readOnly: true + - mountPath: /etc/kubernetes/pki/ca.crt + name: volume-1 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.crt + name: volume-2 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.key + name: volume-3 + readOnly: true + hostNetwork: true + restartPolicy: Always + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: volume-0 + - hostPath: + path: /etc/kubernetes/pki/ca.crt + name: volume-1 + - hostPath: + path: /etc/kubernetes/pki/admin.crt + name: volume-2 + - hostPath: + path: /etc/kubernetes/pki/admin.key + name: volume-3 +status: {} +EOF +docker pull docker.io/prom/prometheus:v2.53.0 +cat </workdir/clusters//manifests/prometheus.yaml +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: prometheus + namespace: kube-system +spec: + containers: + - args: + - --config.file=/etc/prometheus/prometheus.yaml + - --web.listen-address=0.0.0.0:9090 + command: + - prometheus + image: docker.io/prom/prometheus:v2.53.0 + imagePullPolicy: Never + name: prometheus + ports: + - containerPort: 9090 + hostPort: 9090 + resources: {} + volumeMounts: + - mountPath: /etc/prometheus/prometheus.yaml + name: volume-0 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.crt + name: volume-1 + readOnly: true + - mountPath: /etc/kubernetes/pki/admin.key + name: volume-2 + readOnly: true + - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.crt + name: volume-3 + readOnly: true + - mountPath: /etc/kubernetes/pki/apiserver-etcd-client.key + name: volume-4 + readOnly: true + hostNetwork: true + restartPolicy: Always + securityContext: + runAsGroup: 0 + runAsUser: 0 + volumes: + - hostPath: + path: /var/components/prometheus/etc/prometheus/prometheus.yaml + name: volume-0 + - hostPath: + path: /etc/kubernetes/pki/admin.crt + name: volume-1 + - hostPath: + path: /etc/kubernetes/pki/admin.key + name: volume-2 + - hostPath: + path: /etc/kubernetes/pki/apiserver-etcd-client.crt + name: volume-3 + - hostPath: + path: /etc/kubernetes/pki/apiserver-etcd-client.key + name: volume-4 +status: {} +EOF +docker pull docker.io/jaegertracing/all-in-one:1.58.1 +cat </workdir/clusters//manifests/jaeger.yaml +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: jaeger + namespace: kube-system +spec: + containers: + - args: + - --collector.otlp.enabled=true + - --query.http-server.host-port=0.0.0.0:16686 + image: docker.io/jaegertracing/all-in-one:1.58.1 + imagePullPolicy: Never + name: jaeger + ports: + - containerPort: 16686 + hostPort: 16686 + resources: {} + hostNetwork: true + restartPolicy: Always + securityContext: + runAsGroup: 0 + runAsUser: 0 +status: {} +EOF +cat </workdir/clusters//prometheus.yaml +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - follow_redirects: true + enable_http2: true + scheme: http + timeout: 10s + api_version: v2 + static_configs: + - targets: [] +scrape_configs: +- job_name: "etcd" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/apiserver-etcd-client.crt" + key_file: "/etc/kubernetes/pki/apiserver-etcd-client.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:2379 +- job_name: "kube-apiserver" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:6443 +- job_name: "kube-controller-manager" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10257 +- job_name: "kube-scheduler" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:10259 +- job_name: "kwok-controller-metrics-discovery" + http_sd_configs: + - url: http://127.0.0.1:10247/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:10247 +- job_name: "metrics-server" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - 127.0.0.1:4443 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 +EOF +# Save cluster config to /workdir/clusters//kwok.yaml +KIND_EXPERIMENTAL_PROVIDER=docker kind create cluster --config /workdir/clusters//kind.yaml --name kwok- --image docker.io/kindest/node:v1.30.2 --wait 29m +KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image registry.k8s.io/kwok/kwok:v0.6.0 --name kwok- +KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image docker.io/kubernetesui/dashboard:v2.7.0 --name kwok- +KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image docker.io/kubernetesui/metrics-scraper:v1.0.9 --name kwok- +KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image registry.k8s.io/metrics-server/metrics-server:v0.7.1 --name kwok- +KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image docker.io/prom/prometheus:v2.53.0 --name kwok- +KIND_EXPERIMENTAL_PROVIDER=docker kind load docker-image docker.io/jaegertracing/all-in-one:1.58.1 --name kwok- +kubectl config view --minify=true --raw=true +cat </workdir/clusters//kubeconfig.yaml +EOF +kubectl cordon kwok--control-plane +docker exec kwok--control-plane chmod -R +r /etc/kubernetes/pki +# Set up apiservice for metrics server +# Add context kwok- to ~/.kube/config diff --git a/test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster.txt b/test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster.txt new file mode 100644 index 000000000..6adb5adb4 --- /dev/null +++ b/test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster.txt @@ -0,0 +1,65 @@ +# Save cluster config to /workdir/clusters//kwok.yaml +mkdir -p /workdir/clusters/ +mkdir -p /workdir/clusters//pki +# Generate PKI to /workdir/clusters//pki +mkdir -p /workdir/clusters//etcd +nerdctl pull registry.k8s.io/etcd:3.5.11-0 +nerdctl pull registry.k8s.io/kube-apiserver:v1.30.2 +nerdctl pull registry.k8s.io/kube-controller-manager:v1.30.2 +nerdctl pull registry.k8s.io/kube-scheduler:v1.30.2 +nerdctl pull registry.k8s.io/kwok/kwok:v0.6.0 +cat </workdir/clusters//kubeconfig.yaml +apiVersion: v1 +clusters: +- cluster: + certificate-authority: /workdir/clusters//pki/ca.crt + server: https://127.0.0.1:32766 + name: kwok- +contexts: +- context: + cluster: kwok- + user: kwok- + name: kwok- +current-context: kwok- +kind: Config +preferences: {} +users: +- name: kwok- + user: + client-certificate: /workdir/clusters//pki/admin.crt + client-key: /workdir/clusters//pki/admin.key +EOF +cat </workdir/clusters//kubeconfig +apiVersion: v1 +clusters: +- cluster: + certificate-authority: /etc/kubernetes/pki/ca.crt + server: https://kwok--kube-apiserver:6443 + name: kwok- +contexts: +- context: + cluster: kwok- + user: kwok- + name: kwok- +current-context: kwok- +kind: Config +preferences: {} +users: +- name: kwok- + user: + client-certificate: /etc/kubernetes/pki/admin.crt + client-key: /etc/kubernetes/pki/admin.key +EOF +# Save cluster config to /workdir/clusters//kwok.yaml +nerdctl network create kwok- --label=com.docker.compose.project=kwok- +nerdctl create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 +nerdctl create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-apiserver:v1.30.2 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt +nerdctl create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.30.2 --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=/root/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --kube-api-qps=5000 --kube-api-burst=10000 +nerdctl create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-scheduler:v1.30.2 --kubeconfig=/root/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 +nerdctl create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:/root/.kwok/kwok.yaml:ro registry.k8s.io/kwok/kwok:v0.6.0 --manage-all-nodes=true --kubeconfig=/root/.kube/config --config=/root/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=200 +# Add context kwok- to ~/.kube/config +nerdctl start kwok--etcd +nerdctl start kwok--kube-apiserver +nerdctl start kwok--kube-controller-manager +nerdctl start kwok--kube-scheduler +nerdctl start kwok--kwok-controller diff --git a/test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster_with_extra.txt b/test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster_with_extra.txt new file mode 100644 index 000000000..3adc6ec08 --- /dev/null +++ b/test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster_with_extra.txt @@ -0,0 +1,154 @@ +# Save cluster config to /workdir/clusters//kwok.yaml +mkdir -p /workdir/clusters/ +mkdir -p /workdir/clusters//pki +# Generate PKI to /workdir/clusters//pki +mkdir -p /workdir/clusters//etcd +nerdctl pull registry.k8s.io/etcd:3.5.11-0 +nerdctl pull registry.k8s.io/kube-apiserver:v1.30.2 +nerdctl pull registry.k8s.io/kube-controller-manager:v1.30.2 +nerdctl pull registry.k8s.io/kube-scheduler:v1.30.2 +nerdctl pull registry.k8s.io/kwok/kwok:v0.6.0 +nerdctl pull docker.io/prom/prometheus:v2.53.0 +cat </workdir/clusters//prometheus.yaml +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - follow_redirects: true + enable_http2: true + scheme: http + timeout: 10s + api_version: v2 + static_configs: + - targets: [] +scrape_configs: +- job_name: "etcd" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - kwok--etcd:2379 +- job_name: "kube-apiserver" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - kwok--kube-apiserver:6443 +- job_name: "kube-controller-manager" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - kwok--kube-controller-manager:10257 +- job_name: "kube-scheduler" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - kwok--kube-scheduler:10259 +- job_name: "kwok-controller-metrics-discovery" + http_sd_configs: + - url: http://kwok--kwok-controller:10247/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - kwok--kwok-controller:10247 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 +EOF +chmod 0644 /workdir/clusters//prometheus.yaml +cat </workdir/clusters//kubeconfig.yaml +apiVersion: v1 +clusters: +- cluster: + certificate-authority: /workdir/clusters//pki/ca.crt + server: https://127.0.0.1:32766 + name: kwok- +contexts: +- context: + cluster: kwok- + user: kwok- + name: kwok- +current-context: kwok- +kind: Config +preferences: {} +users: +- name: kwok- + user: + client-certificate: /workdir/clusters//pki/admin.crt + client-key: /workdir/clusters//pki/admin.key +EOF +cat </workdir/clusters//kubeconfig +apiVersion: v1 +clusters: +- cluster: + certificate-authority: /etc/kubernetes/pki/ca.crt + server: https://kwok--kube-apiserver:6443 + name: kwok- +contexts: +- context: + cluster: kwok- + user: kwok- + name: kwok- +current-context: kwok- +kind: Config +preferences: {} +users: +- name: kwok- + user: + client-certificate: /etc/kubernetes/pki/admin.crt + client-key: /etc/kubernetes/pki/admin.key +EOF +# Save cluster config to /workdir/clusters//kwok.yaml +nerdctl network create kwok- --label=com.docker.compose.project=kwok- +nerdctl create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug +nerdctl create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/test/e2e/kwokctl/dryrun/extras/apiserver:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.30.2 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt --v=5 +nerdctl create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/test/e2e/kwokctl/dryrun/extras/controller-manager:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.30.2 --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=/root/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=5 +nerdctl create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/test/e2e/kwokctl/dryrun/extras/scheduler:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.30.2 --kubeconfig=/root/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=5 +nerdctl create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:/root/.kwok/kwok.yaml:ro --volume=/test/e2e/kwokctl/dryrun/extras/controller:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kwok/kwok:v0.6.0 --manage-all-nodes=true --kubeconfig=/root/.kube/config --config=/root/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=200 --v=-4 +nerdctl create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/test/e2e/kwokctl/dryrun/extras/prometheus:/extras/tmp --env=TEST_KEY=TEST_VALUE docker.io/prom/prometheus:v2.53.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 --log.level=debug +# Add context kwok- to ~/.kube/config +nerdctl start kwok--etcd +nerdctl start kwok--kube-apiserver +nerdctl start kwok--kube-controller-manager +nerdctl start kwok--kube-scheduler +nerdctl start kwok--kwok-controller +nerdctl start kwok--prometheus diff --git a/test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster_with_verbosity.txt b/test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster_with_verbosity.txt new file mode 100644 index 000000000..9577c210d --- /dev/null +++ b/test/e2e/kwokctl/dryrun/testdata/nerdctl/create_cluster_with_verbosity.txt @@ -0,0 +1,194 @@ +# Save cluster config to /workdir/clusters//kwok.yaml +mkdir -p /workdir/clusters/ +mkdir -p /workdir/clusters//pki +# Generate PKI to /workdir/clusters//pki +mkdir -p /workdir/clusters//logs +touch /workdir/clusters//logs/audit.log +cp /test/kwokctl/audit-policy.yaml /workdir/clusters//audit.yaml +mkdir -p /workdir/clusters//etcd +nerdctl pull registry.k8s.io/etcd:3.5.11-0 +nerdctl pull registry.k8s.io/kube-apiserver:v1.30.2 +cat </workdir/clusters//apiserver-tracing-config.yaml +apiVersion: apiserver.config.k8s.io/v1alpha1 +kind: TracingConfiguration +endpoint: kwok--jaeger:4317 +samplingRatePerMillion: 1000000 +EOF +nerdctl pull registry.k8s.io/kube-controller-manager:v1.30.2 +cp /test/kwokctl/scheduler-config.yaml /workdir/clusters//scheduler.yaml +cat <>/workdir/clusters//scheduler.yaml +clientConnection: + kubeconfig: "/root/.kube/config" +EOF +nerdctl pull registry.k8s.io/kube-scheduler:v1.30.2 +nerdctl pull registry.k8s.io/kwok/kwok:v0.6.0 +nerdctl pull registry.k8s.io/metrics-server/metrics-server:v0.7.1 +nerdctl pull docker.io/prom/prometheus:v2.53.0 +nerdctl pull docker.io/jaegertracing/all-in-one:1.58.1 +nerdctl pull docker.io/kubernetesui/dashboard:v2.7.0 +nerdctl pull docker.io/kubernetesui/metrics-scraper:v1.0.9 +cat </workdir/clusters//prometheus.yaml +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - follow_redirects: true + enable_http2: true + scheme: http + timeout: 10s + api_version: v2 + static_configs: + - targets: [] +scrape_configs: +- job_name: "etcd" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - kwok--etcd:2379 +- job_name: "kube-apiserver" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - kwok--kube-apiserver:6443 +- job_name: "kube-controller-manager" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - kwok--kube-controller-manager:10257 +- job_name: "kube-scheduler" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - kwok--kube-scheduler:10259 +- job_name: "kwok-controller-metrics-discovery" + http_sd_configs: + - url: http://kwok--kwok-controller:10247/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - kwok--kwok-controller:10247 +- job_name: "metrics-server" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - kwok--kwok-controller:4443 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 +EOF +chmod 0644 /workdir/clusters//prometheus.yaml +cat </workdir/clusters//kubeconfig.yaml +apiVersion: v1 +clusters: +- cluster: + certificate-authority: /workdir/clusters//pki/ca.crt + server: https://127.0.0.1:32766 + name: kwok- +contexts: +- context: + cluster: kwok- + user: kwok- + name: kwok- +current-context: kwok- +kind: Config +preferences: {} +users: +- name: kwok- + user: + client-certificate: /workdir/clusters//pki/admin.crt + client-key: /workdir/clusters//pki/admin.key +EOF +cat </workdir/clusters//kubeconfig +apiVersion: v1 +clusters: +- cluster: + certificate-authority: /etc/kubernetes/pki/ca.crt + server: https://kwok--kube-apiserver:6443 + name: kwok- +contexts: +- context: + cluster: kwok- + user: kwok- + name: kwok- +current-context: kwok- +kind: Config +preferences: {} +users: +- name: kwok- + user: + client-certificate: /etc/kubernetes/pki/admin.crt + client-key: /etc/kubernetes/pki/admin.key +EOF +# Save cluster config to /workdir/clusters//kwok.yaml +nerdctl network create kwok- --label=com.docker.compose.project=kwok- +nerdctl create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 +nerdctl create --name=kwok--jaeger --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=16686:16686/tcp docker.io/jaegertracing/all-in-one:1.58.1 --collector.otlp.enabled=true --query.http-server.host-port=0.0.0.0:16686 +nerdctl create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//audit.yaml:/etc/kubernetes/audit-policy.yaml:ro --volume=/workdir/clusters//logs/audit.log:/var/log/kubernetes/audit/audit.log --volume=/workdir/clusters//apiserver-tracing-config.yaml:/etc/kubernetes/apiserver-tracing-config.yaml:ro registry.k8s.io/kube-apiserver:v1.30.2 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt --audit-policy-file=/etc/kubernetes/audit-policy.yaml --audit-log-path=/var/log/kubernetes/audit/audit.log --tracing-config-file=/etc/kubernetes/apiserver-tracing-config.yaml +nerdctl create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.30.2 --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=/root/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 +nerdctl create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//scheduler.yaml:/etc/kubernetes/scheduler.yaml:ro registry.k8s.io/kube-scheduler:v1.30.2 --config=/etc/kubernetes/scheduler.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 +nerdctl create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:/root/.kwok/kwok.yaml:ro registry.k8s.io/kwok/kwok:v0.6.0 --manage-all-nodes=true --kubeconfig=/root/.kube/config --config=/root/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=200 +nerdctl create --name=kwok--dashboard --pull=never --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=8000:8080/tcp --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/kubernetesui/dashboard:v2.7.0 --insecure-bind-address=0.0.0.0 --bind-address=127.0.0.1 --port=0 --enable-insecure-login --enable-skip-login --disable-settings-authorizer --sidecar-host=kwok--dashboard-metrics-scraper:8000 --system-banner=Welcome to kwok- --kubeconfig=/root/.kube/config --insecure-port=8080 +nerdctl create --name=kwok--metrics-server --pull=never --entrypoint=/metrics-server --network=kwok- --user=root --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/metrics-server/metrics-server:v0.7.1 --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --kubelet-use-node-status-port --kubelet-insecure-tls --metric-resolution=15s --bind-address=0.0.0.0 --secure-port=4443 --kubeconfig=/root/.kube/config --authentication-kubeconfig=/root/.kube/config --authorization-kubeconfig=/root/.kube/config --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key +nerdctl create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/prom/prometheus:v2.53.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 +nerdctl create --name=kwok--dashboard-metrics-scraper --pull=never --network=kwok- --user=root --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/kubernetesui/metrics-scraper:v1.0.9 --db-file=/metrics.db --kubeconfig=/root/.kube/config +# Add context kwok- to ~/.kube/config +nerdctl start kwok--etcd +nerdctl start kwok--jaeger +nerdctl start kwok--kube-apiserver +nerdctl start kwok--kube-controller-manager +nerdctl start kwok--kube-scheduler +nerdctl start kwok--kwok-controller +nerdctl start kwok--dashboard +nerdctl start kwok--metrics-server +nerdctl start kwok--prometheus +nerdctl start kwok--dashboard-metrics-scraper +# Set up apiservice for metrics server diff --git a/test/e2e/kwokctl/dryrun/testdata/podman/create_cluster.txt b/test/e2e/kwokctl/dryrun/testdata/podman/create_cluster.txt new file mode 100644 index 000000000..a7de8160e --- /dev/null +++ b/test/e2e/kwokctl/dryrun/testdata/podman/create_cluster.txt @@ -0,0 +1,65 @@ +# Save cluster config to /workdir/clusters//kwok.yaml +mkdir -p /workdir/clusters/ +mkdir -p /workdir/clusters//pki +# Generate PKI to /workdir/clusters//pki +mkdir -p /workdir/clusters//etcd +podman pull registry.k8s.io/etcd:3.5.11-0 +podman pull registry.k8s.io/kube-apiserver:v1.30.2 +podman pull registry.k8s.io/kube-controller-manager:v1.30.2 +podman pull registry.k8s.io/kube-scheduler:v1.30.2 +podman pull registry.k8s.io/kwok/kwok:v0.6.0 +cat </workdir/clusters//kubeconfig.yaml +apiVersion: v1 +clusters: +- cluster: + certificate-authority: /workdir/clusters//pki/ca.crt + server: https://127.0.0.1:32766 + name: kwok- +contexts: +- context: + cluster: kwok- + user: kwok- + name: kwok- +current-context: kwok- +kind: Config +preferences: {} +users: +- name: kwok- + user: + client-certificate: /workdir/clusters//pki/admin.crt + client-key: /workdir/clusters//pki/admin.key +EOF +cat </workdir/clusters//kubeconfig +apiVersion: v1 +clusters: +- cluster: + certificate-authority: /etc/kubernetes/pki/ca.crt + server: https://kwok--kube-apiserver:6443 + name: kwok- +contexts: +- context: + cluster: kwok- + user: kwok- + name: kwok- +current-context: kwok- +kind: Config +preferences: {} +users: +- name: kwok- + user: + client-certificate: /etc/kubernetes/pki/admin.crt + client-key: /etc/kubernetes/pki/admin.key +EOF +# Save cluster config to /workdir/clusters//kwok.yaml +podman network create kwok- --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- +podman create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 +podman create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --requires=kwok--etcd --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-apiserver:v1.30.2 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt +podman create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.30.2 --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=/root/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --kube-api-qps=5000 --kube-api-burst=10000 +podman create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-scheduler:v1.30.2 --kubeconfig=/root/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 +podman create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:/root/.kwok/kwok.yaml:ro registry.k8s.io/kwok/kwok:v0.6.0 --manage-all-nodes=true --kubeconfig=/root/.kube/config --config=/root/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=200 +# Add context kwok- to ~/.kube/config +podman start kwok--etcd +podman start kwok--kube-apiserver +podman start kwok--kube-controller-manager +podman start kwok--kube-scheduler +podman start kwok--kwok-controller diff --git a/test/e2e/kwokctl/dryrun/testdata/podman/create_cluster_with_extra.txt b/test/e2e/kwokctl/dryrun/testdata/podman/create_cluster_with_extra.txt new file mode 100644 index 000000000..7ceac656b --- /dev/null +++ b/test/e2e/kwokctl/dryrun/testdata/podman/create_cluster_with_extra.txt @@ -0,0 +1,154 @@ +# Save cluster config to /workdir/clusters//kwok.yaml +mkdir -p /workdir/clusters/ +mkdir -p /workdir/clusters//pki +# Generate PKI to /workdir/clusters//pki +mkdir -p /workdir/clusters//etcd +podman pull registry.k8s.io/etcd:3.5.11-0 +podman pull registry.k8s.io/kube-apiserver:v1.30.2 +podman pull registry.k8s.io/kube-controller-manager:v1.30.2 +podman pull registry.k8s.io/kube-scheduler:v1.30.2 +podman pull registry.k8s.io/kwok/kwok:v0.6.0 +podman pull docker.io/prom/prometheus:v2.53.0 +cat </workdir/clusters//prometheus.yaml +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - follow_redirects: true + enable_http2: true + scheme: http + timeout: 10s + api_version: v2 + static_configs: + - targets: [] +scrape_configs: +- job_name: "etcd" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - kwok--etcd:2379 +- job_name: "kube-apiserver" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - kwok--kube-apiserver:6443 +- job_name: "kube-controller-manager" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - kwok--kube-controller-manager:10257 +- job_name: "kube-scheduler" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - kwok--kube-scheduler:10259 +- job_name: "kwok-controller-metrics-discovery" + http_sd_configs: + - url: http://kwok--kwok-controller:10247/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - kwok--kwok-controller:10247 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 +EOF +chmod 0644 /workdir/clusters//prometheus.yaml +cat </workdir/clusters//kubeconfig.yaml +apiVersion: v1 +clusters: +- cluster: + certificate-authority: /workdir/clusters//pki/ca.crt + server: https://127.0.0.1:32766 + name: kwok- +contexts: +- context: + cluster: kwok- + user: kwok- + name: kwok- +current-context: kwok- +kind: Config +preferences: {} +users: +- name: kwok- + user: + client-certificate: /workdir/clusters//pki/admin.crt + client-key: /workdir/clusters//pki/admin.key +EOF +cat </workdir/clusters//kubeconfig +apiVersion: v1 +clusters: +- cluster: + certificate-authority: /etc/kubernetes/pki/ca.crt + server: https://kwok--kube-apiserver:6443 + name: kwok- +contexts: +- context: + cluster: kwok- + user: kwok- + name: kwok- +current-context: kwok- +kind: Config +preferences: {} +users: +- name: kwok- + user: + client-certificate: /etc/kubernetes/pki/admin.crt + client-key: /etc/kubernetes/pki/admin.key +EOF +# Save cluster config to /workdir/clusters//kwok.yaml +podman network create kwok- --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- +podman create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug +podman create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --requires=kwok--etcd --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/test/e2e/kwokctl/dryrun/extras/apiserver:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.30.2 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt --v=5 +podman create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/test/e2e/kwokctl/dryrun/extras/controller-manager:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.30.2 --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=/root/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=5 +podman create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/test/e2e/kwokctl/dryrun/extras/scheduler:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.30.2 --kubeconfig=/root/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=5 +podman create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:/root/.kwok/kwok.yaml:ro --volume=/test/e2e/kwokctl/dryrun/extras/controller:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kwok/kwok:v0.6.0 --manage-all-nodes=true --kubeconfig=/root/.kube/config --config=/root/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=200 --v=-4 +podman create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --requires=kwok--etcd --requires=kwok--kube-apiserver --requires=kwok--kube-controller-manager --requires=kwok--kube-scheduler --requires=kwok--kwok-controller --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/test/e2e/kwokctl/dryrun/extras/prometheus:/extras/tmp --env=TEST_KEY=TEST_VALUE docker.io/prom/prometheus:v2.53.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 --log.level=debug +# Add context kwok- to ~/.kube/config +podman start kwok--etcd +podman start kwok--kube-apiserver +podman start kwok--kube-controller-manager +podman start kwok--kube-scheduler +podman start kwok--kwok-controller +podman start kwok--prometheus diff --git a/test/e2e/kwokctl/dryrun/testdata/podman/create_cluster_with_verbosity.txt b/test/e2e/kwokctl/dryrun/testdata/podman/create_cluster_with_verbosity.txt new file mode 100644 index 000000000..b8ceba94d --- /dev/null +++ b/test/e2e/kwokctl/dryrun/testdata/podman/create_cluster_with_verbosity.txt @@ -0,0 +1,194 @@ +# Save cluster config to /workdir/clusters//kwok.yaml +mkdir -p /workdir/clusters/ +mkdir -p /workdir/clusters//pki +# Generate PKI to /workdir/clusters//pki +mkdir -p /workdir/clusters//logs +touch /workdir/clusters//logs/audit.log +cp /test/kwokctl/audit-policy.yaml /workdir/clusters//audit.yaml +mkdir -p /workdir/clusters//etcd +podman pull registry.k8s.io/etcd:3.5.11-0 +podman pull registry.k8s.io/kube-apiserver:v1.30.2 +cat </workdir/clusters//apiserver-tracing-config.yaml +apiVersion: apiserver.config.k8s.io/v1alpha1 +kind: TracingConfiguration +endpoint: kwok--jaeger:4317 +samplingRatePerMillion: 1000000 +EOF +podman pull registry.k8s.io/kube-controller-manager:v1.30.2 +cp /test/kwokctl/scheduler-config.yaml /workdir/clusters//scheduler.yaml +cat <>/workdir/clusters//scheduler.yaml +clientConnection: + kubeconfig: "/root/.kube/config" +EOF +podman pull registry.k8s.io/kube-scheduler:v1.30.2 +podman pull registry.k8s.io/kwok/kwok:v0.6.0 +podman pull registry.k8s.io/metrics-server/metrics-server:v0.7.1 +podman pull docker.io/prom/prometheus:v2.53.0 +podman pull docker.io/jaegertracing/all-in-one:1.58.1 +podman pull docker.io/kubernetesui/dashboard:v2.7.0 +podman pull docker.io/kubernetesui/metrics-scraper:v1.0.9 +cat </workdir/clusters//prometheus.yaml +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - follow_redirects: true + enable_http2: true + scheme: http + timeout: 10s + api_version: v2 + static_configs: + - targets: [] +scrape_configs: +- job_name: "etcd" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - kwok--etcd:2379 +- job_name: "kube-apiserver" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - kwok--kube-apiserver:6443 +- job_name: "kube-controller-manager" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - kwok--kube-controller-manager:10257 +- job_name: "kube-scheduler" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - kwok--kube-scheduler:10259 +- job_name: "kwok-controller-metrics-discovery" + http_sd_configs: + - url: http://kwok--kwok-controller:10247/discovery/prometheus +- job_name: "kwok-controller" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - kwok--kwok-controller:10247 +- job_name: "metrics-server" + scheme: https + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + tls_config: + cert_file: "/etc/kubernetes/pki/admin.crt" + key_file: "/etc/kubernetes/pki/admin.key" + insecure_skip_verify: true + static_configs: + - targets: + - kwok--kwok-controller:4443 +- job_name: "prometheus" + scheme: http + honor_timestamps: true + metrics_path: /metrics + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - 127.0.0.1:9090 +EOF +chmod 0644 /workdir/clusters//prometheus.yaml +cat </workdir/clusters//kubeconfig.yaml +apiVersion: v1 +clusters: +- cluster: + certificate-authority: /workdir/clusters//pki/ca.crt + server: https://127.0.0.1:32766 + name: kwok- +contexts: +- context: + cluster: kwok- + user: kwok- + name: kwok- +current-context: kwok- +kind: Config +preferences: {} +users: +- name: kwok- + user: + client-certificate: /workdir/clusters//pki/admin.crt + client-key: /workdir/clusters//pki/admin.key +EOF +cat </workdir/clusters//kubeconfig +apiVersion: v1 +clusters: +- cluster: + certificate-authority: /etc/kubernetes/pki/ca.crt + server: https://kwok--kube-apiserver:6443 + name: kwok- +contexts: +- context: + cluster: kwok- + user: kwok- + name: kwok- +current-context: kwok- +kind: Config +preferences: {} +users: +- name: kwok- + user: + client-certificate: /etc/kubernetes/pki/admin.crt + client-key: /etc/kubernetes/pki/admin.key +EOF +# Save cluster config to /workdir/clusters//kwok.yaml +podman network create kwok- --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- +podman create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 +podman create --name=kwok--jaeger --pull=never --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=16686:16686/tcp docker.io/jaegertracing/all-in-one:1.58.1 --collector.otlp.enabled=true --query.http-server.host-port=0.0.0.0:16686 +podman create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --requires=kwok--etcd --requires=kwok--jaeger --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//audit.yaml:/etc/kubernetes/audit-policy.yaml:ro --volume=/workdir/clusters//logs/audit.log:/var/log/kubernetes/audit/audit.log --volume=/workdir/clusters//apiserver-tracing-config.yaml:/etc/kubernetes/apiserver-tracing-config.yaml:ro registry.k8s.io/kube-apiserver:v1.30.2 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt --audit-policy-file=/etc/kubernetes/audit-policy.yaml --audit-log-path=/var/log/kubernetes/audit/audit.log --tracing-config-file=/etc/kubernetes/apiserver-tracing-config.yaml +podman create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/kube-controller-manager:v1.30.2 --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=/root/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 +podman create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//scheduler.yaml:/etc/kubernetes/scheduler.yaml:ro registry.k8s.io/kube-scheduler:v1.30.2 --config=/etc/kubernetes/scheduler.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 +podman create --name=kwok--kwok-controller --pull=never --entrypoint=kwok --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/workdir/clusters//kwok.yaml:/root/.kwok/kwok.yaml:ro registry.k8s.io/kwok/kwok:v0.6.0 --manage-all-nodes=true --kubeconfig=/root/.kube/config --config=/root/.kwok/kwok.yaml --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --node-ip= --node-name=kwok--kwok-controller --node-port=10247 --server-address=0.0.0.0:10247 --node-lease-duration-seconds=200 +podman create --name=kwok--dashboard --pull=never --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=8000:8080/tcp --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/kubernetesui/dashboard:v2.7.0 --insecure-bind-address=0.0.0.0 --bind-address=127.0.0.1 --port=0 --enable-insecure-login --enable-skip-login --disable-settings-authorizer --sidecar-host=kwok--dashboard-metrics-scraper:8000 --system-banner=Welcome to kwok- --kubeconfig=/root/.kube/config --insecure-port=8080 +podman create --name=kwok--metrics-server --pull=never --entrypoint=/metrics-server --network=kwok- --user=root --requires=kwok--kwok-controller --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro registry.k8s.io/metrics-server/metrics-server:v0.7.1 --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --kubelet-use-node-status-port --kubelet-insecure-tls --metric-resolution=15s --bind-address=0.0.0.0 --secure-port=4443 --kubeconfig=/root/.kube/config --authentication-kubeconfig=/root/.kube/config --authorization-kubeconfig=/root/.kube/config --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key +podman create --name=kwok--prometheus --pull=never --entrypoint=prometheus --network=kwok- --requires=kwok--etcd --requires=kwok--kube-apiserver --requires=kwok--kube-controller-manager --requires=kwok--kube-scheduler --requires=kwok--kwok-controller --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=9090:9090/tcp --volume=/workdir/clusters//prometheus.yaml:/etc/prometheus/prometheus.yaml:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/prom/prometheus:v2.53.0 --config.file=/etc/prometheus/prometheus.yaml --web.listen-address=0.0.0.0:9090 +podman create --name=kwok--dashboard-metrics-scraper --pull=never --network=kwok- --user=root --requires=kwok--metrics-server --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:/root/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro docker.io/kubernetesui/metrics-scraper:v1.0.9 --db-file=/metrics.db --kubeconfig=/root/.kube/config +# Add context kwok- to ~/.kube/config +podman start kwok--etcd +podman start kwok--jaeger +podman start kwok--kube-apiserver +podman start kwok--kube-controller-manager +podman start kwok--kube-scheduler +podman start kwok--kwok-controller +podman start kwok--dashboard +podman start kwok--metrics-server +podman start kwok--prometheus +podman start kwok--dashboard-metrics-scraper +# Set up apiservice for metrics server