From f1fb25c1305f9144c2803d6376f8e22ab7fc84aa Mon Sep 17 00:00:00 2001 From: Shiming Zhang Date: Sun, 7 Apr 2024 17:32:52 +0800 Subject: [PATCH] Structured Kind configuration --- pkg/kwokctl/runtime/kind/cluster.go | 4 +- .../kind/config/kind/v1alpha4/types.go | 322 +++++++++++++ .../kind/config/kubeadm/v1beta3/types.go | 447 ++++++++++++++++++ pkg/kwokctl/runtime/kind/kind.go | 286 ++++++++++- pkg/kwokctl/runtime/kind/kind.yaml.tpl | 198 -------- .../docker/create_cluster_with_extra.txt | 2 +- test/kwokctl/testdata/extra.yaml | 6 - .../testdata/kind-podman/create_cluster.txt | 61 +-- .../kind-podman/create_cluster_with_extra.txt | 147 +++--- .../create_cluster_with_verbosity.txt | 154 +++--- test/kwokctl/testdata/kind/create_cluster.txt | 61 +-- .../kind/create_cluster_with_extra.txt | 147 +++--- .../kind/create_cluster_with_verbosity.txt | 154 +++--- .../nerdctl/create_cluster_with_extra.txt | 2 +- .../podman/create_cluster_with_extra.txt | 2 +- 15 files changed, 1381 insertions(+), 612 deletions(-) create mode 100644 pkg/kwokctl/runtime/kind/config/kind/v1alpha4/types.go create mode 100644 pkg/kwokctl/runtime/kind/config/kubeadm/v1beta3/types.go delete mode 100644 pkg/kwokctl/runtime/kind/kind.yaml.tpl diff --git a/pkg/kwokctl/runtime/kind/cluster.go b/pkg/kwokctl/runtime/kind/cluster.go index 497580d1c5..cd641bc02f 100644 --- a/pkg/kwokctl/runtime/kind/cluster.go +++ b/pkg/kwokctl/runtime/kind/cluster.go @@ -309,10 +309,10 @@ func (c *Cluster) addKind(ctx context.Context, env *env) (err error) { var featureGates []string var runtimeConfig []string if conf.KubeFeatureGates != "" { - featureGates = strings.Split(strings.ReplaceAll(conf.KubeFeatureGates, "=", ": "), ",") + featureGates = strings.Split(conf.KubeFeatureGates, ",") } if conf.KubeRuntimeConfig != "" { - runtimeConfig = strings.Split(strings.ReplaceAll(conf.KubeRuntimeConfig, "=", ": "), ",") + runtimeConfig = strings.Split(conf.KubeRuntimeConfig, ",") } pkiPath := c.GetWorkdirPath(runtime.PkiName) diff --git a/pkg/kwokctl/runtime/kind/config/kind/v1alpha4/types.go b/pkg/kwokctl/runtime/kind/config/kind/v1alpha4/types.go new file mode 100644 index 0000000000..b6a7464507 --- /dev/null +++ b/pkg/kwokctl/runtime/kind/config/kind/v1alpha4/types.go @@ -0,0 +1,322 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha4 copy from https://github.com/kubernetes-sigs/kind/blob/master/pkg/apis/config/v1alpha4/types.go +package v1alpha4 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Cluster contains kind cluster configuration +type Cluster struct { + metav1.TypeMeta `json:",inline"` + + // The cluster name. + // Optional, this will be overridden by --name / KIND_CLUSTER_NAME + Name string `yaml:"name,omitempty" json:"name,omitempty"` + + // Nodes contains the list of nodes defined in the `kind` Cluster + // If unset this will default to a single control-plane node + // Note that if more than one control plane is specified, an external + // control plane load balancer will be provisioned implicitly + Nodes []Node `yaml:"nodes,omitempty" json:"nodes,omitempty"` + + /* Advanced fields */ + + // Networking contains cluster wide network settings + Networking Networking `yaml:"networking,omitempty" json:"networking,omitempty"` + + // FeatureGates contains a map of Kubernetes feature gates to whether they + // are enabled. The feature gates specified here are passed to all Kubernetes components as flags or in config. + // + // https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/ + FeatureGates map[string]bool `yaml:"featureGates,omitempty" json:"featureGates,omitempty"` + + // RuntimeConfig Keys and values are translated into --runtime-config values for kube-apiserver, separated by commas. + // + // Use this to enable alpha APIs. + RuntimeConfig map[string]string `yaml:"runtimeConfig,omitempty" json:"runtimeConfig,omitempty"` + + // KubeadmConfigPatches are applied to the generated kubeadm config as + // merge patches. The `kind` field must match the target object, and + // if `apiVersion` is specified it will only be applied to matching objects. + // + // This should be an inline yaml blob-string + // + // https://tools.ietf.org/html/rfc7386 + // + // The cluster-level patches are applied before the node-level patches. + KubeadmConfigPatches []string `yaml:"kubeadmConfigPatches,omitempty" json:"kubeadmConfigPatches,omitempty"` + + // KubeadmConfigPatchesJSON6902 are applied to the generated kubeadm config + // as JSON 6902 patches. The `kind` field must match the target object, and + // if group or version are specified it will only be objects matching the + // apiVersion: group+"/"+version + // + // Name and Namespace are now ignored, but the fields continue to exist for + // backwards compatibility of parsing the config. The name of the generated + // config was/is always fixed as is the namespace so these fields have + // always been a no-op. + // + // https://tools.ietf.org/html/rfc6902 + // + // The cluster-level patches are applied before the node-level patches. + KubeadmConfigPatchesJSON6902 []PatchJSON6902 `yaml:"kubeadmConfigPatchesJSON6902,omitempty" json:"kubeadmConfigPatchesJSON6902,omitempty"` + + // ContainerdConfigPatches are applied to every node's containerd config + // in the order listed. + // These should be toml stringsto be applied as merge patches + ContainerdConfigPatches []string `yaml:"containerdConfigPatches,omitempty" json:"containerdConfigPatches,omitempty"` + + // ContainerdConfigPatchesJSON6902 are applied to every node's containerd config + // in the order listed. + // These should be YAML or JSON formatting RFC 6902 JSON patches + ContainerdConfigPatchesJSON6902 []string `yaml:"containerdConfigPatchesJSON6902,omitempty" json:"containerdConfigPatchesJSON6902,omitempty"` +} + +// TypeMeta partially copies apimachinery/pkg/apis/meta/v1.TypeMeta +// No need for a direct dependence; the fields are stable. +type TypeMeta struct { + Kind string `yaml:"kind,omitempty" json:"kind,omitempty"` + APIVersion string `yaml:"apiVersion,omitempty" json:"apiVersion,omitempty"` +} + +// Node contains settings for a node in the `kind` Cluster. +// A node in kind config represent a container that will be provisioned with all the components +// required for the assigned role in the Kubernetes cluster +type Node struct { + // Role defines the role of the node in the Kubernetes cluster + // created by kind + // + // Defaults to "control-plane" + Role NodeRole `yaml:"role,omitempty" json:"role,omitempty"` + + // Image is the node image to use when creating this node + // If unset a default image will be used, see defaults.Image + Image string `yaml:"image,omitempty" json:"image,omitempty"` + + // Labels are the labels with which the respective node will be labeled + Labels map[string]string `yaml:"labels,omitempty" json:"labels,omitempty"` + + /* Advanced fields */ + + // TODO: cri-like types should be inline instead + // ExtraMounts describes additional mount points for the node container + // These may be used to bind a hostPath + ExtraMounts []Mount `yaml:"extraMounts,omitempty" json:"extraMounts,omitempty"` + + // ExtraPortMappings describes additional port mappings for the node container + // binded to a host Port + ExtraPortMappings []PortMapping `yaml:"extraPortMappings,omitempty" json:"extraPortMappings,omitempty"` + + // KubeadmConfigPatches are applied to the generated kubeadm config as + // merge patches. The `kind` field must match the target object, and + // if `apiVersion` is specified it will only be applied to matching objects. + // + // This should be an inline yaml blob-string + // + // https://tools.ietf.org/html/rfc7386 + // + // The node-level patches will be applied after the cluster-level patches + // have been applied. (See Cluster.KubeadmConfigPatches) + KubeadmConfigPatches []string `yaml:"kubeadmConfigPatches,omitempty" json:"kubeadmConfigPatches,omitempty"` + + // KubeadmConfigPatchesJSON6902 are applied to the generated kubeadm config + // as JSON 6902 patches. The `kind` field must match the target object, and + // if group or version are specified it will only be objects matching the + // apiVersion: group+"/"+version + // + // Name and Namespace are now ignored, but the fields continue to exist for + // backwards compatibility of parsing the config. The name of the generated + // config was/is always fixed as is the namespace so these fields have + // always been a no-op. + // + // https://tools.ietf.org/html/rfc6902 + // + // The node-level patches will be applied after the cluster-level patches + // have been applied. (See Cluster.KubeadmConfigPatchesJSON6902) + KubeadmConfigPatchesJSON6902 []PatchJSON6902 `yaml:"kubeadmConfigPatchesJSON6902,omitempty" json:"kubeadmConfigPatchesJSON6902,omitempty"` +} + +// NodeRole defines possible role for nodes in a Kubernetes cluster managed by `kind` +type NodeRole string + +const ( + // ControlPlaneRole identifies a node that hosts a Kubernetes control-plane. + // NOTE: in single node clusters, control-plane nodes act also as a worker + // nodes, in which case the taint will be removed. see: + // https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#control-plane-node-isolation + ControlPlaneRole NodeRole = "control-plane" + // WorkerRole identifies a node that hosts a Kubernetes worker + WorkerRole NodeRole = "worker" +) + +// Networking contains cluster wide network settings +type Networking struct { + // IPFamily is the network cluster model, currently it can be ipv4 or ipv6 + IPFamily ClusterIPFamily `yaml:"ipFamily,omitempty" json:"ipFamily,omitempty"` + // APIServerPort is the listen port on the host for the Kubernetes API Server + // Defaults to a random port on the host obtained by kind + // + // NOTE: if you set the special value of `-1` then the node backend + // (docker, podman...) will be left to pick the port instead. + // This is potentially useful for remote hosts, BUT it means when the container + // is restarted it will be randomized. Leave this unset to allow kind to pick it. + APIServerPort int32 `yaml:"apiServerPort,omitempty" json:"apiServerPort,omitempty"` + // APIServerAddress is the listen address on the host for the Kubernetes + // API Server. This should be an IP address. + // + // Defaults to 127.0.0.1 + APIServerAddress string `yaml:"apiServerAddress,omitempty" json:"apiServerAddress,omitempty"` + // PodSubnet is the CIDR used for pod IPs + // kind will select a default if unspecified + PodSubnet string `yaml:"podSubnet,omitempty" json:"podSubnet,omitempty"` + // ServiceSubnet is the CIDR used for services VIPs + // kind will select a default if unspecified for IPv6 + ServiceSubnet string `yaml:"serviceSubnet,omitempty" json:"serviceSubnet,omitempty"` + // If DisableDefaultCNI is true, kind will not install the default CNI setup. + // Instead the user should install their own CNI after creating the cluster. + DisableDefaultCNI bool `yaml:"disableDefaultCNI,omitempty" json:"disableDefaultCNI,omitempty"` + // KubeProxyMode defines if kube-proxy should operate in iptables or ipvs mode + // Defaults to 'iptables' mode + KubeProxyMode ProxyMode `yaml:"kubeProxyMode,omitempty" json:"kubeProxyMode,omitempty"` + // DNSSearch defines the DNS search domain to use for nodes. If not set, this will be inherited from the host. + DNSSearch *[]string `yaml:"dnsSearch,omitempty" json:"dnsSearch,omitempty"` +} + +// ClusterIPFamily defines cluster network IP family +type ClusterIPFamily string + +const ( + // IPv4Family sets ClusterIPFamily to ipv4 + IPv4Family ClusterIPFamily = "ipv4" + // IPv6Family sets ClusterIPFamily to ipv6 + IPv6Family ClusterIPFamily = "ipv6" + // DualStackFamily sets ClusterIPFamily to dual + DualStackFamily ClusterIPFamily = "dual" +) + +// ProxyMode defines a proxy mode for kube-proxy +type ProxyMode string + +const ( + // IPTablesProxyMode sets ProxyMode to iptables + IPTablesProxyMode ProxyMode = "iptables" + // IPVSProxyMode sets ProxyMode to ipvs + IPVSProxyMode ProxyMode = "ipvs" +) + +// PatchJSON6902 represents an inline kustomize json 6902 patch +// https://tools.ietf.org/html/rfc6902 +type PatchJSON6902 struct { + // these fields specify the patch target resource + Group string `yaml:"group" json:"group"` + Version string `yaml:"version" json:"version"` + Kind string `yaml:"kind" json:"kind"` + // Patch should contain the contents of the json patch as a string + Patch string `yaml:"patch" json:"patch"` +} + +/* +These types are from +https://github.com/kubernetes/kubernetes/blob/063e7ff358fdc8b0916e6f39beedc0d025734cb1/pkg/kubelet/apis/cri/runtime/v1alpha2/api.pb.go#L183 +*/ + +// Mount specifies a host volume to mount into a container. +// This is a close copy of the upstream cri Mount type +// see: k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2 +// It additionally serializes the "propagation" field with the string enum +// names on disk as opposed to the int32 values, and the serialized field names +// have been made closer to core/v1 VolumeMount field names +// In yaml this looks like: +// +// containerPath: /foo +// hostPath: /bar +// readOnly: true +// selinuxRelabel: false +// propagation: None +// +// Propagation may be one of: None, HostToContainer, Bidirectional +type Mount struct { + // Path of the mount within the container. + ContainerPath string `yaml:"containerPath,omitempty" json:"containerPath,omitempty"` + // Path of the mount on the host. If the hostPath doesn't exist, then runtimes + // should report error. If the hostpath is a symbolic link, runtimes should + // follow the symlink and mount the real destination to container. + HostPath string `yaml:"hostPath,omitempty" json:"hostPath,omitempty"` + // If set, the mount is read-only. + Readonly bool `yaml:"readOnly,omitempty" json:"readOnly,omitempty"` + // If set, the mount needs SELinux relabeling. + SelinuxRelabel bool `yaml:"selinuxRelabel,omitempty" json:"selinuxRelabel,omitempty"` + // Requested propagation mode. + Propagation MountPropagation `yaml:"propagation,omitempty" json:"propagation,omitempty"` +} + +// PortMapping specifies a host port mapped into a container port. +// In yaml this looks like: +// +// containerPort: 80 +// hostPort: 8000 +// listenAddress: 127.0.0.1 +// protocol: TCP +type PortMapping struct { + // Port within the container. + ContainerPort int32 `yaml:"containerPort,omitempty" json:"containerPort,omitempty"` + // Port on the host. + // + // If unset, a random port will be selected. + // + // NOTE: if you set the special value of `-1` then the node backend + // (docker, podman...) will be left to pick the port instead. + // This is potentially useful for remote hosts, BUT it means when the container + // is restarted it will be randomized. Leave this unset to allow kind to pick it. + HostPort int32 `yaml:"hostPort,omitempty" json:"hostPort,omitempty"` + // TODO: add protocol (tcp/udp) and port-ranges + ListenAddress string `yaml:"listenAddress,omitempty" json:"listenAddress,omitempty"` + // Protocol (TCP/UDP/SCTP) + Protocol PortMappingProtocol `yaml:"protocol,omitempty" json:"protocol,omitempty"` +} + +// MountPropagation represents an "enum" for mount propagation options, +// see also Mount. +type MountPropagation string + +const ( + // MountPropagationNone specifies that no mount propagation + // ("private" in Linux terminology). + MountPropagationNone MountPropagation = "None" + // MountPropagationHostToContainer specifies that mounts get propagated + // from the host to the container ("rslave" in Linux). + MountPropagationHostToContainer MountPropagation = "HostToContainer" + // MountPropagationBidirectional specifies that mounts get propagated from + // the host to the container and from the container to the host + // ("rshared" in Linux). + MountPropagationBidirectional MountPropagation = "Bidirectional" +) + +// PortMappingProtocol represents an "enum" for port mapping protocol options, +// see also PortMapping. +type PortMappingProtocol string + +const ( + // PortMappingProtocolTCP specifies TCP protocol + PortMappingProtocolTCP PortMappingProtocol = "TCP" + // PortMappingProtocolUDP specifies UDP protocol + PortMappingProtocolUDP PortMappingProtocol = "UDP" + // PortMappingProtocolSCTP specifies SCTP protocol + PortMappingProtocolSCTP PortMappingProtocol = "SCTP" +) diff --git a/pkg/kwokctl/runtime/kind/config/kubeadm/v1beta3/types.go b/pkg/kwokctl/runtime/kind/config/kubeadm/v1beta3/types.go new file mode 100644 index 0000000000..2b27b6870d --- /dev/null +++ b/pkg/kwokctl/runtime/kind/config/kubeadm/v1beta3/types.go @@ -0,0 +1,447 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1beta3 copy from https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/apis/kubeadm/v1beta4/types.go +package v1beta3 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// InitConfiguration contains a list of elements that is specific "kubeadm init"-only runtime +// information. +type InitConfiguration struct { + metav1.TypeMeta `json:",inline"` + + // `kubeadm init`-only information. These fields are solely used the first time `kubeadm init` runs. + // After that, the information in the fields IS NOT uploaded to the `kubeadm-config` ConfigMap + // that is used by `kubeadm upgrade` for instance. These fields must be omitempty. + + // BootstrapTokens is respected at `kubeadm init` time and describes a set of Bootstrap Tokens to create. + // This information IS NOT uploaded to the kubeadm cluster configmap, partly because of its sensitive nature + // +optional + // BootstrapTokens []bootstraptokenv1.BootstrapToken `json:"bootstrapTokens,omitempty"` + + // NodeRegistration holds fields that relate to registering the new control-plane node to the cluster + // +optional + NodeRegistration NodeRegistrationOptions `json:"nodeRegistration,omitempty"` + + // LocalAPIEndpoint represents the endpoint of the API server instance that's deployed on this control plane node + // In HA setups, this differs from ClusterConfiguration.ControlPlaneEndpoint in the sense that ControlPlaneEndpoint + // is the global endpoint for the cluster, which then loadbalances the requests to each individual API server. This + // configuration object lets you customize what IP/DNS name and port the local API server advertises it's accessible + // on. By default, kubeadm tries to auto-detect the IP of the default interface and use that, but in case that process + // fails you may set the desired value here. + // +optional + LocalAPIEndpoint APIEndpoint `json:"localAPIEndpoint,omitempty"` + + // CertificateKey sets the key with which certificates and keys are encrypted prior to being uploaded in + // a secret in the cluster during the uploadcerts init phase. + // The certificate key is a hex encoded string that is an AES key of size 32 bytes. + // +optional + CertificateKey string `json:"certificateKey,omitempty"` + + // SkipPhases is a list of phases to skip during command execution. + // The list of phases can be obtained with the "kubeadm init --help" command. + // The flag "--skip-phases" takes precedence over this field. + // +optional + SkipPhases []string `json:"skipPhases,omitempty"` + + // Patches contains options related to applying patches to components deployed by kubeadm during + // "kubeadm init". + // +optional + Patches *Patches `json:"patches,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// ClusterConfiguration contains cluster-wide configuration for a kubeadm cluster +type ClusterConfiguration struct { + metav1.TypeMeta `json:",inline"` + + // Etcd holds configuration for etcd. + // +optional + Etcd Etcd `json:"etcd,omitempty"` + + // Networking holds configuration for the networking topology of the cluster. + // +optional + Networking Networking `json:"networking,omitempty"` + + // KubernetesVersion is the target version of the control plane. + // +optional + KubernetesVersion string `json:"kubernetesVersion,omitempty"` + + // ControlPlaneEndpoint sets a stable IP address or DNS name for the control plane; it + // can be a valid IP address or a RFC-1123 DNS subdomain, both with optional TCP port. + // In case the ControlPlaneEndpoint is not specified, the AdvertiseAddress + BindPort + // are used; in case the ControlPlaneEndpoint is specified but without a TCP port, + // the BindPort is used. + // Possible usages are: + // e.g. In a cluster with more than one control plane instances, this field should be + // assigned the address of the external load balancer in front of the + // control plane instances. + // e.g. in environments with enforced node recycling, the ControlPlaneEndpoint + // could be used for assigning a stable DNS to the control plane. + // +optional + ControlPlaneEndpoint string `json:"controlPlaneEndpoint,omitempty"` + + // APIServer contains extra settings for the API server control plane component + // +optional + APIServer APIServer `json:"apiServer,omitempty"` + + // ControllerManager contains extra settings for the controller manager control plane component + // +optional + ControllerManager ControlPlaneComponent `json:"controllerManager,omitempty"` + + // Scheduler contains extra settings for the scheduler control plane component + // +optional + Scheduler ControlPlaneComponent `json:"scheduler,omitempty"` + + // DNS defines the options for the DNS add-on installed in the cluster. + // +optional + DNS DNS `json:"dns,omitempty"` + + // CertificatesDir specifies where to store or look for all required certificates. + // +optional + CertificatesDir string `json:"certificatesDir,omitempty"` + + // ImageRepository sets the container registry to pull images from. + // If empty, `registry.k8s.io` will be used by default; in case of kubernetes version is a CI build (kubernetes version starts with `ci/`) + // `gcr.io/k8s-staging-ci-images` will be used as a default for control plane components and for kube-proxy, while `registry.k8s.io` + // will be used for all the other images. + // +optional + ImageRepository string `json:"imageRepository,omitempty"` + + // FeatureGates enabled by the user. + // +optional + FeatureGates map[string]bool `json:"featureGates,omitempty"` + + // The cluster name + // +optional + ClusterName string `json:"clusterName,omitempty"` +} + +// ControlPlaneComponent holds settings common to control plane component of the cluster +type ControlPlaneComponent struct { + // ExtraArgs is an extra set of flags to pass to the control plane component. + // A key in this map is the flag name as it appears on the + // command line except without leading dash(es). + // TODO: This is temporary and ideally we would like to switch all components to + // use ComponentConfig + ConfigMaps. + // +optional + ExtraArgs map[string]string `json:"extraArgs,omitempty"` + + // ExtraVolumes is an extra set of host volumes, mounted to the control plane component. + // +optional + ExtraVolumes []HostPathMount `json:"extraVolumes,omitempty"` +} + +// APIServer holds settings necessary for API server deployments in the cluster +type APIServer struct { + ControlPlaneComponent `json:",inline"` + + // CertSANs sets extra Subject Alternative Names for the API Server signing cert. + // +optional + CertSANs []string `json:"certSANs,omitempty"` + + // TimeoutForControlPlane controls the timeout that we use for API server to appear + // +optional + TimeoutForControlPlane *metav1.Duration `json:"timeoutForControlPlane,omitempty"` +} + +// DNSAddOnType defines string identifying DNS add-on types +type DNSAddOnType string + +// DNS defines the DNS addon that should be used in the cluster +type DNS struct { + // ImageMeta allows to customize the image used for the DNS component + ImageMeta `json:",inline"` +} + +// ImageMeta allows to customize the image used for components that are not +// originated from the Kubernetes/Kubernetes release process +type ImageMeta struct { + // ImageRepository sets the container registry to pull images from. + // if not set, the ImageRepository defined in ClusterConfiguration will be used instead. + // +optional + ImageRepository string `json:"imageRepository,omitempty"` + + // ImageTag allows to specify a tag for the image. + // In case this value is set, kubeadm does not change automatically the version of the above components during upgrades. + // +optional + ImageTag string `json:"imageTag,omitempty"` + + //TODO: evaluate if we need also a ImageName based on user feedbacks +} + +// APIEndpoint struct contains elements of API server instance deployed on a node. +type APIEndpoint struct { + // AdvertiseAddress sets the IP address for the API server to advertise. + // +optional + AdvertiseAddress string `json:"advertiseAddress,omitempty"` + + // BindPort sets the secure port for the API Server to bind to. + // Defaults to 6443. + // +optional + BindPort int32 `json:"bindPort,omitempty"` +} + +// NodeRegistrationOptions holds fields that relate to registering a new control-plane or node to the cluster, either via "kubeadm init" or "kubeadm join" +type NodeRegistrationOptions struct { + + // Name is the `.Metadata.Name` field of the Node API object that will be created in this `kubeadm init` or `kubeadm join` operation. + // This field is also used in the CommonName field of the kubelet's client certificate to the API server. + // Defaults to the hostname of the node if not provided. + // +optional + Name string `json:"name,omitempty"` + + // CRISocket is used to retrieve container runtime info. This information will be annotated to the Node API object, for later re-use + // +optional + CRISocket string `json:"criSocket,omitempty"` + + // Taints specifies the taints the Node API object should be registered with. If this field is unset, i.e. nil, + // it will be defaulted with a control-plane taint for control-plane nodes. If you don't want to taint your control-plane + // node, set this field to an empty slice, i.e. `taints: []` in the YAML file. This field is solely used for Node registration. + Taints []corev1.Taint `json:"taints"` + + // KubeletExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file + // kubeadm writes at runtime for the kubelet to source. This overrides the generic base-level configuration in the kubelet-config ConfigMap + // Flags have higher priority when parsing. These values are local and specific to the node kubeadm is executing on. + // A key in this map is the flag name as it appears on the + // command line except without leading dash(es). + // +optional + KubeletExtraArgs map[string]string `json:"kubeletExtraArgs,omitempty"` + + // IgnorePreflightErrors provides a slice of pre-flight errors to be ignored when the current node is registered, e.g. 'IsPrivilegedUser,Swap'. + // Value 'all' ignores errors from all checks. + // +optional + IgnorePreflightErrors []string `json:"ignorePreflightErrors,omitempty"` + + // ImagePullPolicy specifies the policy for image pulling during kubeadm "init" and "join" operations. + // The value of this field must be one of "Always", "IfNotPresent" or "Never". + // If this field is unset kubeadm will default it to "IfNotPresent", or pull the required images if not present on the host. + // +optional + ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"` +} + +// Networking contains elements describing cluster's networking configuration +type Networking struct { + // ServiceSubnet is the subnet used by k8s services. Defaults to "10.96.0.0/12". + // +optional + ServiceSubnet string `json:"serviceSubnet,omitempty"` + // PodSubnet is the subnet used by pods. + // +optional + PodSubnet string `json:"podSubnet,omitempty"` + // DNSDomain is the dns domain used by k8s services. Defaults to "cluster.local". + // +optional + DNSDomain string `json:"dnsDomain,omitempty"` +} + +// Etcd contains elements describing Etcd configuration. +type Etcd struct { + + // Local provides configuration knobs for configuring the local etcd instance + // Local and External are mutually exclusive + // +optional + Local *LocalEtcd `json:"local,omitempty"` + + // External describes how to connect to an external etcd cluster + // Local and External are mutually exclusive + // +optional + External *ExternalEtcd `json:"external,omitempty"` +} + +// LocalEtcd describes that kubeadm should run an etcd cluster locally +type LocalEtcd struct { + // ImageMeta allows to customize the container used for etcd + ImageMeta `json:",inline"` + + // DataDir is the directory etcd will place its data. + // Defaults to "/var/lib/etcd". + DataDir string `json:"dataDir"` + + // ExtraArgs are extra arguments provided to the etcd binary + // when run inside a static pod. + // A key in this map is the flag name as it appears on the + // command line except without leading dash(es). + // +optional + ExtraArgs map[string]string `json:"extraArgs,omitempty"` + + // ServerCertSANs sets extra Subject Alternative Names for the etcd server signing cert. + // +optional + ServerCertSANs []string `json:"serverCertSANs,omitempty"` + // PeerCertSANs sets extra Subject Alternative Names for the etcd peer signing cert. + // +optional + PeerCertSANs []string `json:"peerCertSANs,omitempty"` +} + +// ExternalEtcd describes an external etcd cluster. +// Kubeadm has no knowledge of where certificate files live and they must be supplied. +type ExternalEtcd struct { + // Endpoints of etcd members. Required for ExternalEtcd. + Endpoints []string `json:"endpoints"` + + // CAFile is an SSL Certificate Authority file used to secure etcd communication. + // Required if using a TLS connection. + CAFile string `json:"caFile"` + + // CertFile is an SSL certification file used to secure etcd communication. + // Required if using a TLS connection. + CertFile string `json:"certFile"` + + // KeyFile is an SSL key file used to secure etcd communication. + // Required if using a TLS connection. + KeyFile string `json:"keyFile"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// JoinConfiguration contains elements describing a particular node. +type JoinConfiguration struct { + metav1.TypeMeta `json:",inline"` + + // NodeRegistration holds fields that relate to registering the new control-plane node to the cluster + // +optional + NodeRegistration NodeRegistrationOptions `json:"nodeRegistration,omitempty"` + + // CACertPath is the path to the SSL certificate authority used to + // secure comunications between node and control-plane. + // Defaults to "/etc/kubernetes/pki/ca.crt". + // +optional + CACertPath string `json:"caCertPath,omitempty"` + + // Discovery specifies the options for the kubelet to use during the TLS Bootstrap process + Discovery Discovery `json:"discovery"` + + // ControlPlane defines the additional control plane instance to be deployed on the joining node. + // If nil, no additional control plane instance will be deployed. + // +optional + ControlPlane *JoinControlPlane `json:"controlPlane,omitempty"` + + // SkipPhases is a list of phases to skip during command execution. + // The list of phases can be obtained with the "kubeadm join --help" command. + // The flag "--skip-phases" takes precedence over this field. + // +optional + SkipPhases []string `json:"skipPhases,omitempty"` + + // Patches contains options related to applying patches to components deployed by kubeadm during + // "kubeadm join". + // +optional + Patches *Patches `json:"patches,omitempty"` +} + +// JoinControlPlane contains elements describing an additional control plane instance to be deployed on the joining node. +type JoinControlPlane struct { + // LocalAPIEndpoint represents the endpoint of the API server instance to be deployed on this node. + // +optional + LocalAPIEndpoint APIEndpoint `json:"localAPIEndpoint,omitempty"` + + // CertificateKey is the key that is used for decryption of certificates after they are downloaded from the secret + // upon joining a new control plane node. The corresponding encryption key is in the InitConfiguration. + // The certificate key is a hex encoded string that is an AES key of size 32 bytes. + // +optional + CertificateKey string `json:"certificateKey,omitempty"` +} + +// Discovery specifies the options for the kubelet to use during the TLS Bootstrap process +type Discovery struct { + // BootstrapToken is used to set the options for bootstrap token based discovery + // BootstrapToken and File are mutually exclusive + // +optional + BootstrapToken *BootstrapTokenDiscovery `json:"bootstrapToken,omitempty"` + + // File is used to specify a file or URL to a kubeconfig file from which to load cluster information + // BootstrapToken and File are mutually exclusive + // +optional + File *FileDiscovery `json:"file,omitempty"` + + // TLSBootstrapToken is a token used for TLS bootstrapping. + // If .BootstrapToken is set, this field is defaulted to .BootstrapToken.Token, but can be overridden. + // If .File is set, this field **must be set** in case the KubeConfigFile does not contain any other authentication information + // +optional + TLSBootstrapToken string `json:"tlsBootstrapToken,omitempty" datapolicy:"token"` + + // Timeout modifies the discovery timeout + // +optional + Timeout *metav1.Duration `json:"timeout,omitempty"` +} + +// BootstrapTokenDiscovery is used to set the options for bootstrap token based discovery +type BootstrapTokenDiscovery struct { + // Token is a token used to validate cluster information + // fetched from the control-plane. + Token string `json:"token" datapolicy:"token"` + + // APIServerEndpoint is an IP or domain name to the API server from which info will be fetched. + // +optional + APIServerEndpoint string `json:"apiServerEndpoint,omitempty"` + + // CACertHashes specifies a set of public key pins to verify + // when token-based discovery is used. The root CA found during discovery + // must match one of these values. Specifying an empty set disables root CA + // pinning, which can be unsafe. Each hash is specified as ":", + // where the only currently supported type is "sha256". This is a hex-encoded + // SHA-256 hash of the Subject Public Key Info (SPKI) object in DER-encoded + // ASN.1. These hashes can be calculated using, for example, OpenSSL. + // +optional + CACertHashes []string `json:"caCertHashes,omitempty" datapolicy:"security-key"` + + // UnsafeSkipCAVerification allows token-based discovery + // without CA verification via CACertHashes. This can weaken + // the security of kubeadm since other nodes can impersonate the control-plane. + // +optional + UnsafeSkipCAVerification bool `json:"unsafeSkipCAVerification,omitempty"` +} + +// FileDiscovery is used to specify a file or URL to a kubeconfig file from which to load cluster information +type FileDiscovery struct { + // KubeConfigPath is used to specify the actual file path or URL to the kubeconfig file from which to load cluster information + KubeConfigPath string `json:"kubeConfigPath"` +} + +// HostPathMount contains elements describing volumes that are mounted from the +// host. +type HostPathMount struct { + // Name of the volume inside the pod template. + Name string `json:"name"` + // HostPath is the path in the host that will be mounted inside + // the pod. + HostPath string `json:"hostPath"` + // MountPath is the path inside the pod where hostPath will be mounted. + MountPath string `json:"mountPath"` + // ReadOnly controls write access to the volume + // +optional + ReadOnly bool `json:"readOnly,omitempty"` + // PathType is the type of the HostPath. + // +optional + PathType corev1.HostPathType `json:"pathType,omitempty"` +} + +// Patches contains options related to applying patches to components deployed by kubeadm. +type Patches struct { + // Directory is a path to a directory that contains files named "target[suffix][+patchtype].extension". + // For example, "kube-apiserver0+merge.yaml" or just "etcd.json". "target" can be one of + // "kube-apiserver", "kube-controller-manager", "kube-scheduler", "etcd", "kubeletconfiguration". + // "patchtype" can be one of "strategic" "merge" or "json" and they match the patch formats supported by kubectl. + // The default "patchtype" is "strategic". "extension" must be either "json" or "yaml". + // "suffix" is an optional string that can be used to determine which patches are applied + // first alpha-numerically. + // +optional + Directory string `json:"directory,omitempty"` +} diff --git a/pkg/kwokctl/runtime/kind/kind.go b/pkg/kwokctl/runtime/kind/kind.go index 52ff008858..4dbe94e1ee 100644 --- a/pkg/kwokctl/runtime/kind/kind.go +++ b/pkg/kwokctl/runtime/kind/kind.go @@ -17,29 +17,25 @@ limitations under the License. package kind import ( - "bytes" "fmt" - "text/template" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/kwok/pkg/apis/internalversion" "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/kwokctl/runtime" + kindv1alpha4 "sigs.k8s.io/kwok/pkg/kwokctl/runtime/kind/config/kind/v1alpha4" + kubeadmv1beta4 "sigs.k8s.io/kwok/pkg/kwokctl/runtime/kind/config/kubeadm/v1beta3" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/format" "sigs.k8s.io/kwok/pkg/utils/version" - - _ "embed" + "sigs.k8s.io/kwok/pkg/utils/yaml" ) -//go:embed kind.yaml.tpl -var kindYamlTpl string - -var kindYamlTemplate = template.Must(template.New("kind_config").Parse(kindYamlTpl)) - // BuildKind builds the kind yaml content. func BuildKind(conf BuildKindConfig) (string, error) { - buf := bytes.NewBuffer(nil) - var err error conf, err = expendExtrasForBuildKind(conf) if err != nil { @@ -51,11 +47,26 @@ func BuildKind(conf BuildKindConfig) (string, error) { return "", fmt.Errorf("failed to expand host volume paths: %w", err) } - err = kindYamlTemplate.Execute(buf, conf) + kubeadmConfig, err := buildKubeadmConfigV1beta3(conf) + if err != nil { + return "", fmt.Errorf("failed to build kubeadm config: %w", err) + } + kubeadmConfigData, err := yaml.Marshal(kubeadmConfig) + if err != nil { + return "", err + } + + kindConfig, err := buildKindConfigV1alpha4(conf) if err != nil { - return "", fmt.Errorf("failed to execute kind yaml template: %w", err) + return "", fmt.Errorf("failed to build kind config: %w", err) } - return buf.String(), nil + kindConfig.KubeadmConfigPatches = []string{string(kubeadmConfigData)} + kindConfigData, err := yaml.Marshal(kindConfig) + if err != nil { + return "", err + } + + return string(kindConfigData), nil } func expendExtrasForBuildKind(conf BuildKindConfig) (BuildKindConfig, error) { @@ -133,7 +144,7 @@ func expendExtrasForBuildKind(conf BuildKindConfig) (BuildKindConfig, error) { if conf.KubeVersion.LT(version.NewVersion(1, 22, 0)) { return conf, fmt.Errorf("the kube-apiserver version is less than 1.22.0, so the --jaeger-port cannot be enabled") } else if conf.KubeVersion.LT(version.NewVersion(1, 27, 0)) { - conf.FeatureGates = append(conf.FeatureGates, "APIServerTracing: true") + conf.FeatureGates = append(conf.FeatureGates, "APIServerTracing=true") } } @@ -283,3 +294,248 @@ type BuildKindConfig struct { DisableQPSLimits bool KubeVersion version.Version } + +func buildKindConfigV1alpha4(conf BuildKindConfig) (*kindv1alpha4.Cluster, error) { + var extraPortMappings []kindv1alpha4.PortMapping + + if conf.KubeApiserverInsecurePort != 0 { + extraPortMappings = append(extraPortMappings, kindv1alpha4.PortMapping{ + ContainerPort: 8001, + HostPort: int32(conf.KubeApiserverInsecurePort), + Protocol: kindv1alpha4.PortMappingProtocolTCP, + }) + } + + if conf.DashboardPort != 0 { + extraPortMappings = append(extraPortMappings, kindv1alpha4.PortMapping{ + ContainerPort: 8080, + HostPort: int32(conf.DashboardPort), + Protocol: kindv1alpha4.PortMappingProtocolTCP, + }) + } + + if conf.PrometheusPort != 0 { + extraPortMappings = append(extraPortMappings, kindv1alpha4.PortMapping{ + ContainerPort: 9090, + HostPort: int32(conf.PrometheusPort), + Protocol: kindv1alpha4.PortMappingProtocolTCP, + }) + } + + if conf.JaegerPort != 0 { + extraPortMappings = append(extraPortMappings, kindv1alpha4.PortMapping{ + ContainerPort: 16686, + HostPort: int32(conf.JaegerPort), + Protocol: kindv1alpha4.PortMappingProtocolTCP, + }) + } + + if conf.KwokControllerPort != 0 { + extraPortMappings = append(extraPortMappings, kindv1alpha4.PortMapping{ + ContainerPort: 10247, + HostPort: int32(conf.KwokControllerPort), + Protocol: kindv1alpha4.PortMappingProtocolTCP, + }) + } + + if conf.EtcdPort != 0 { + extraPortMappings = append(extraPortMappings, kindv1alpha4.PortMapping{ + ContainerPort: 2379, + HostPort: int32(conf.EtcdPort), + Protocol: kindv1alpha4.PortMappingProtocolTCP, + }) + } + + var extraMounts = []kindv1alpha4.Mount{ + { + HostPath: conf.Workdir, + ContainerPath: "/etc/kwok/", + }, + { + HostPath: fmt.Sprintf("%s/manifests", conf.Workdir), + ContainerPath: "/etc/kubernetes/manifests", + }, + { + HostPath: fmt.Sprintf("%s/pki", conf.Workdir), + ContainerPath: "/etc/kubernetes/pki", + }, + } + + for _, vol := range conf.EtcdExtraVolumes { + extraMounts = append(extraMounts, kindv1alpha4.Mount{ + HostPath: vol.HostPath, + ContainerPath: fmt.Sprintf("/var/components/etcd%s", vol.MountPath), + Readonly: vol.ReadOnly, + }) + } + + for _, vol := range conf.ApiserverExtraVolumes { + extraMounts = append(extraMounts, kindv1alpha4.Mount{ + HostPath: vol.HostPath, + ContainerPath: fmt.Sprintf("/var/components/apiserver%s", vol.MountPath), + Readonly: vol.ReadOnly, + }) + } + + for _, vol := range conf.ControllerManagerExtraVolumes { + extraMounts = append(extraMounts, kindv1alpha4.Mount{ + HostPath: vol.HostPath, + ContainerPath: fmt.Sprintf("/var/components/controller-manager%s", vol.MountPath), + Readonly: vol.ReadOnly, + }) + } + + for _, vol := range conf.SchedulerExtraVolumes { + extraMounts = append(extraMounts, kindv1alpha4.Mount{ + HostPath: vol.HostPath, + ContainerPath: fmt.Sprintf("/var/components/scheduler%s", vol.MountPath), + Readonly: vol.ReadOnly, + }) + } + + for _, vol := range conf.KwokControllerExtraVolumes { + extraMounts = append(extraMounts, kindv1alpha4.Mount{ + HostPath: vol.HostPath, + ContainerPath: fmt.Sprintf("/var/components/controller%s", vol.MountPath), + Readonly: vol.ReadOnly, + }) + } + + for _, vol := range conf.PrometheusExtraVolumes { + extraMounts = append(extraMounts, kindv1alpha4.Mount{ + HostPath: vol.HostPath, + ContainerPath: fmt.Sprintf("/var/components/prometheus%s", vol.MountPath), + Readonly: vol.ReadOnly, + }) + } + + featureGates := map[string]bool{} + for _, fg := range conf.FeatureGates { + kv := strings.SplitN(fg, "=", 2) + if len(kv) == 2 { + featureGates[kv[0]] = kv[1] == "true" + } else { + return nil, fmt.Errorf("invalid feature gate: %s", fg) + } + } + + runtimeConfig := map[string]string{} + for _, rc := range conf.RuntimeConfig { + kv := strings.SplitN(rc, "=", 2) + if len(kv) == 2 { + runtimeConfig[kv[0]] = kv[1] + } else { + return nil, fmt.Errorf("invalid runtime config: %s", rc) + } + } + + c := kindv1alpha4.Cluster{ + TypeMeta: metav1.TypeMeta{ + Kind: "Cluster", + APIVersion: "kind.x-k8s.io/v1alpha4", + }, + FeatureGates: featureGates, + RuntimeConfig: runtimeConfig, + + Networking: kindv1alpha4.Networking{ + APIServerPort: int32(conf.KubeApiserverPort), + }, + + Nodes: []kindv1alpha4.Node{ + { + Role: kindv1alpha4.ControlPlaneRole, + ExtraPortMappings: extraPortMappings, + ExtraMounts: extraMounts, + }, + }, + } + + return &c, nil +} + +func buildKubeadmConfigV1beta3(conf BuildKindConfig) (*kubeadmv1beta4.ClusterConfiguration, error) { + c := kubeadmv1beta4.ClusterConfiguration{ + TypeMeta: metav1.TypeMeta{ + Kind: "ClusterConfiguration", + APIVersion: "kubeadm.k8s.io/v1beta3", + }, + Etcd: kubeadmv1beta4.Etcd{ + Local: &kubeadmv1beta4.LocalEtcd{ + DataDir: "/var/lib/etcd", + }, + }, + } + + if len(conf.EtcdExtraArgs) > 0 { + c.Etcd.Local.ExtraArgs = map[string]string{} + for _, arg := range conf.EtcdExtraArgs { + c.Etcd.Local.ExtraArgs[arg.Key] = arg.Value + } + } + + if len(conf.EtcdExtraVolumes) > 0 { + return nil, fmt.Errorf("etcd extra volumes are not supported") + } + + if len(conf.ApiserverExtraArgs) > 0 { + c.APIServer.ExtraArgs = map[string]string{} + for _, arg := range conf.ApiserverExtraArgs { + c.APIServer.ExtraArgs[arg.Key] = arg.Value + } + } + + if len(conf.ApiserverExtraVolumes) > 0 { + c.APIServer.ExtraVolumes = []kubeadmv1beta4.HostPathMount{} + for _, vol := range conf.ApiserverExtraVolumes { + c.APIServer.ExtraVolumes = append(c.APIServer.ExtraVolumes, kubeadmv1beta4.HostPathMount{ + Name: vol.Name, + HostPath: vol.HostPath, + MountPath: vol.MountPath, + ReadOnly: vol.ReadOnly, + PathType: corev1.HostPathType(vol.PathType), + }) + } + } + + if len(conf.ControllerManagerExtraArgs) > 0 { + c.ControllerManager.ExtraArgs = map[string]string{} + for _, arg := range conf.ControllerManagerExtraArgs { + c.ControllerManager.ExtraArgs[arg.Key] = arg.Value + } + } + + if len(conf.ControllerManagerExtraVolumes) > 0 { + c.ControllerManager.ExtraVolumes = []kubeadmv1beta4.HostPathMount{} + for _, vol := range conf.ControllerManagerExtraVolumes { + c.ControllerManager.ExtraVolumes = append(c.ControllerManager.ExtraVolumes, kubeadmv1beta4.HostPathMount{ + Name: vol.Name, + HostPath: vol.HostPath, + MountPath: vol.MountPath, + ReadOnly: vol.ReadOnly, + PathType: corev1.HostPathType(vol.PathType), + }) + } + } + + if len(conf.SchedulerExtraArgs) > 0 { + c.Scheduler.ExtraArgs = map[string]string{} + for _, arg := range conf.SchedulerExtraArgs { + c.Scheduler.ExtraArgs[arg.Key] = arg.Value + } + } + + if len(conf.SchedulerExtraVolumes) > 0 { + c.Scheduler.ExtraVolumes = []kubeadmv1beta4.HostPathMount{} + for _, vol := range conf.SchedulerExtraVolumes { + c.Scheduler.ExtraVolumes = append(c.Scheduler.ExtraVolumes, kubeadmv1beta4.HostPathMount{ + Name: vol.Name, + HostPath: vol.HostPath, + MountPath: vol.MountPath, + ReadOnly: vol.ReadOnly, + PathType: corev1.HostPathType(vol.PathType), + }) + } + } + + return &c, nil +} diff --git a/pkg/kwokctl/runtime/kind/kind.yaml.tpl b/pkg/kwokctl/runtime/kind/kind.yaml.tpl deleted file mode 100644 index cb5cbea5fb..0000000000 --- a/pkg/kwokctl/runtime/kind/kind.yaml.tpl +++ /dev/null @@ -1,198 +0,0 @@ -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 - -networking: -{{ if .KubeApiserverPort }} - apiServerPort: {{ .KubeApiserverPort }} -{{ end }} -nodes: -- role: control-plane - - {{ if or .KubeApiserverInsecurePort .DashboardPort .PrometheusPort .KwokControllerPort .EtcdPort .JaegerPort}} - extraPortMappings: - {{ if .KubeApiserverInsecurePort }} - - containerPort: 8001 - hostPort: {{ .KubeApiserverInsecurePort }} - protocol: TCP - {{ end }} - {{ if .DashboardPort }} - - containerPort: 8080 - hostPort: {{ .DashboardPort }} - protocol: TCP - {{ end }} - {{ if .PrometheusPort }} - - containerPort: 9090 - hostPort: {{ .PrometheusPort }} - protocol: TCP - {{ end }} - {{ if .JaegerPort }} - - containerPort: 16686 - hostPort: {{ .JaegerPort }} - protocol: TCP - {{ end }} - {{ if .KwokControllerPort }} - - containerPort: 10247 - hostPort: {{ .KwokControllerPort }} - protocol: TCP - {{ end }} - {{ if .EtcdPort }} - - containerPort: 2379 - hostPort: {{ .EtcdPort }} - protocol: TCP - {{ end }} - {{ end }} - - kubeadmConfigPatches: - - {{ if or .EtcdExtraArgs .EtcdExtraVolumes }} - - | - kind: ClusterConfiguration - etcd: - local: - {{ if .EtcdExtraArgs }} - extraArgs: - {{ range .EtcdExtraArgs }} - "{{ .Key }}": "{{ .Value }}" - {{ end }} - {{ end }} - - {{ if .EtcdExtraVolumes }} - extraVolumes: - {{ range .EtcdExtraVolumes }} - - name: {{ .Name }} - hostPath: /var/components/etcd{{ .MountPath }} - mountPath: {{ .MountPath }} - readOnly: {{ .ReadOnly }} - pathType: {{ .PathType }} - {{ end }} - {{ end }} - - {{ end }} - - {{ if or .ApiserverExtraArgs .ApiserverExtraVolumes }} - - | - kind: ClusterConfiguration - apiServer: - {{ if .ApiserverExtraArgs }} - extraArgs: - {{ range .ApiserverExtraArgs }} - "{{ .Key }}": "{{ .Value }}" - {{ end }} - {{ end }} - - {{ if .ApiserverExtraVolumes }} - extraVolumes: - {{ range .ApiserverExtraVolumes }} - - name: {{ .Name }} - hostPath: /var/components/apiserver{{ .MountPath }} - mountPath: {{ .MountPath }} - readOnly: {{ .ReadOnly }} - pathType: {{ .PathType }} - {{ end }} - {{ end }} - {{ end }} - - {{ if or .ControllerManagerExtraArgs .ControllerManagerExtraVolumes }} - - | - kind: ClusterConfiguration - controllerManager: - {{ if .ControllerManagerExtraArgs }} - extraArgs: - {{ range .ControllerManagerExtraArgs }} - "{{ .Key }}": "{{ .Value }}" - {{ end }} - {{ end }} - - {{ if .ControllerManagerExtraVolumes }} - extraVolumes: - {{ range .ControllerManagerExtraVolumes }} - - name: {{ .Name }} - hostPath: /var/components/controller-manager{{ .MountPath }} - mountPath: {{ .MountPath }} - readOnly: {{ .ReadOnly }} - pathType: {{ .PathType }} - {{ end }} - {{ end }} - {{ end }} - - {{ if or .SchedulerExtraArgs .SchedulerExtraVolumes }} - - | - kind: ClusterConfiguration - scheduler: - {{ if .SchedulerExtraArgs }} - extraArgs: - {{ range .SchedulerExtraArgs }} - "{{ .Key }}": "{{ .Value }}" - {{ end }} - {{ end }} - - {{ if .SchedulerExtraVolumes }} - extraVolumes: - {{ range .SchedulerExtraVolumes }} - - name: {{ .Name }} - hostPath: /var/components/scheduler{{ .MountPath }} - mountPath: {{ .MountPath }} - readOnly: {{ .ReadOnly }} - pathType: {{ .PathType }} - {{ end }} - {{ end }} - {{ end }} - - # mount the local file on the control plane - extraMounts: - - hostPath: {{ .Workdir }} - containerPath: /etc/kwok/ - - hostPath: {{ .Workdir }}/manifests - containerPath: /etc/kubernetes/manifests - - hostPath: {{ .Workdir }}/pki - containerPath: /etc/kubernetes/pki - - {{ range .EtcdExtraVolumes }} - - hostPath: {{ .HostPath }} - containerPath: /var/components/etcd{{ .MountPath }} - readOnly: {{ .ReadOnly }} - {{ end }} - - {{ range .ApiserverExtraVolumes }} - - hostPath: {{ .HostPath }} - containerPath: /var/components/apiserver{{ .MountPath }} - readOnly: {{ .ReadOnly }} - {{ end }} - - {{ range .ControllerManagerExtraVolumes }} - - hostPath: {{ .HostPath }} - containerPath: /var/components/controller-manager{{ .MountPath }} - readOnly: {{ .ReadOnly }} - {{ end }} - - {{ range .SchedulerExtraVolumes }} - - hostPath: {{ .HostPath }} - containerPath: /var/components/scheduler{{ .MountPath }} - readOnly: {{ .ReadOnly }} - {{ end }} - - {{ range .KwokControllerExtraVolumes }} - - hostPath: {{ .HostPath }} - containerPath: /var/components/controller{{ .MountPath }} - readOnly: {{ .ReadOnly }} - {{ end }} - - {{ range .PrometheusExtraVolumes }} - - hostPath: {{ .HostPath }} - containerPath: /var/components/prometheus{{ .MountPath }} - readOnly: {{ .ReadOnly }} - {{ end }} - -{{ if .FeatureGates }} -featureGates: -{{ range .FeatureGates }} -- {{ . }} -{{ end }} -{{ end }} - -{{ if .RuntimeConfig }} -runtimeConfig: -{{ range .RuntimeConfig }} -- {{ . }} -{{ end }} -{{ end }} diff --git a/test/kwokctl/testdata/docker/create_cluster_with_extra.txt b/test/kwokctl/testdata/docker/create_cluster_with_extra.txt index 6733b96862..ce346037ab 100644 --- a/test/kwokctl/testdata/docker/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/docker/create_cluster_with_extra.txt @@ -139,7 +139,7 @@ users: EOF # Save cluster config to /workdir/clusters//kwok.yaml docker network create kwok- --label=com.docker.compose.project=kwok- -docker create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp --volume=/extras/etcd:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug +docker create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug docker create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --link=kwok--etcd --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/extras/apiserver:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.29.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt --v=5 docker create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/extras/controller-manager:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.29.0 --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=5 docker create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --link=kwok--kube-apiserver --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/extras/scheduler:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.29.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=5 diff --git a/test/kwokctl/testdata/extra.yaml b/test/kwokctl/testdata/extra.yaml index a081cb4c8e..413167cd60 100644 --- a/test/kwokctl/testdata/extra.yaml +++ b/test/kwokctl/testdata/extra.yaml @@ -58,12 +58,6 @@ componentsPatches: extraArgs: - key: log-level value: "debug" - extraVolumes: - - name: tmp-etcd - hostPath: ./extras/etcd - mountPath: /extras/tmp - readOnly: false - pathType: DirectoryOrCreate extraEnvs: - name: TEST_KEY value: TEST_VALUE diff --git a/test/kwokctl/testdata/kind-podman/create_cluster.txt b/test/kwokctl/testdata/kind-podman/create_cluster.txt index bc6756e34d..b8ef44bfa4 100644 --- a/test/kwokctl/testdata/kind-podman/create_cluster.txt +++ b/test/kwokctl/testdata/kind-podman/create_cluster.txt @@ -7,44 +7,45 @@ podman pull docker.io/kindest/node:v1.29.0 mkdir -p /workdir/clusters//pki mkdir -p /workdir/clusters//manifests cat </workdir/clusters//kind.yaml -kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +kubeadmConfigPatches: +- | + apiServer: + extraArgs: + enable-priority-and-fairness: "false" + max-mutating-requests-inflight: "0" + max-requests-inflight: "0" + apiVersion: kubeadm.k8s.io/v1beta3 + controllerManager: + extraArgs: + kube-api-burst: "10000" + kube-api-qps: "5000" + dns: {} + etcd: + local: + dataDir: /var/lib/etcd + kind: ClusterConfiguration + networking: {} + scheduler: + extraArgs: + kube-api-burst: "10000" + kube-api-qps: "5000" networking: apiServerPort: 32766 nodes: -- role: control-plane +- extraMounts: + - containerPath: /etc/kwok/ + hostPath: /workdir/clusters/ + - containerPath: /etc/kubernetes/manifests + hostPath: /workdir/clusters//manifests + - containerPath: /etc/kubernetes/pki + hostPath: /workdir/clusters//pki extraPortMappings: - containerPort: 2379 hostPort: 32765 protocol: TCP - kubeadmConfigPatches: - - | - kind: ClusterConfiguration - apiServer: - extraArgs: - "max-requests-inflight": "0" - "max-mutating-requests-inflight": "0" - "enable-priority-and-fairness": "false" - - | - kind: ClusterConfiguration - controllerManager: - extraArgs: - "kube-api-qps": "5000" - "kube-api-burst": "10000" - - | - kind: ClusterConfiguration - scheduler: - extraArgs: - "kube-api-qps": "5000" - "kube-api-burst": "10000" - # mount the local file on the control plane - extraMounts: - - hostPath: /workdir/clusters/ - containerPath: /etc/kwok/ - - hostPath: /workdir/clusters//manifests - containerPath: /etc/kubernetes/manifests - - hostPath: /workdir/clusters//pki - containerPath: /etc/kubernetes/pki + role: control-plane EOF podman pull localhost/kwok:test cat </workdir/clusters//manifests/kwok-controller.yaml diff --git a/test/kwokctl/testdata/kind-podman/create_cluster_with_extra.txt b/test/kwokctl/testdata/kind-podman/create_cluster_with_extra.txt index e46db49aa7..b1909752c8 100644 --- a/test/kwokctl/testdata/kind-podman/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/kind-podman/create_cluster_with_extra.txt @@ -7,12 +7,72 @@ podman pull docker.io/kindest/node:v1.29.0 mkdir -p /workdir/clusters//pki mkdir -p /workdir/clusters//manifests cat </workdir/clusters//kind.yaml -kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +kubeadmConfigPatches: +- | + apiServer: + extraArgs: + enable-priority-and-fairness: "false" + max-mutating-requests-inflight: "0" + max-requests-inflight: "0" + v: "5" + extraVolumes: + - hostPath: /extras/apiserver + mountPath: /extras/tmp + name: tmp-apiserver + pathType: DirectoryOrCreate + apiVersion: kubeadm.k8s.io/v1beta3 + controllerManager: + extraArgs: + kube-api-burst: "10000" + kube-api-qps: "5000" + v: "5" + extraVolumes: + - hostPath: /extras/controller-manager + mountPath: /extras/tmp + name: tmp-controller-manager + pathType: DirectoryOrCreate + dns: {} + etcd: + local: + dataDir: /var/lib/etcd + extraArgs: + log-level: debug + kind: ClusterConfiguration + networking: {} + scheduler: + extraArgs: + kube-api-burst: "10000" + kube-api-qps: "5000" + v: "5" + extraVolumes: + - hostPath: /extras/scheduler + mountPath: /extras/tmp + name: tmp-scheduler + pathType: DirectoryOrCreate networking: apiServerPort: 32766 nodes: -- role: control-plane +- extraMounts: + - containerPath: /etc/kwok/ + hostPath: /workdir/clusters/ + - containerPath: /etc/kubernetes/manifests + hostPath: /workdir/clusters//manifests + - containerPath: /etc/kubernetes/pki + hostPath: /workdir/clusters//pki + - containerPath: /var/components/apiserver/extras/tmp + hostPath: /extras/apiserver + - containerPath: /var/components/controller-manager/extras/tmp + hostPath: /extras/controller-manager + - containerPath: /var/components/scheduler/extras/tmp + hostPath: /extras/scheduler + - containerPath: /var/components/controller/extras/tmp + hostPath: /extras/controller + - containerPath: /var/components/prometheus/extras/tmp + hostPath: /extras/prometheus + - containerPath: /var/components/prometheus/etc/prometheus/prometheus.yaml + hostPath: /workdir/clusters//prometheus.yaml extraPortMappings: - containerPort: 9090 hostPort: 9090 @@ -20,88 +80,7 @@ nodes: - containerPort: 2379 hostPort: 32765 protocol: TCP - kubeadmConfigPatches: - - | - kind: ClusterConfiguration - etcd: - local: - extraArgs: - "log-level": "debug" - extraVolumes: - - name: tmp-etcd - hostPath: /var/components/etcd/extras/tmp - mountPath: /extras/tmp - readOnly: false - pathType: DirectoryOrCreate - - | - kind: ClusterConfiguration - apiServer: - extraArgs: - "v": "5" - "max-requests-inflight": "0" - "max-mutating-requests-inflight": "0" - "enable-priority-and-fairness": "false" - extraVolumes: - - name: tmp-apiserver - hostPath: /var/components/apiserver/extras/tmp - mountPath: /extras/tmp - readOnly: false - pathType: DirectoryOrCreate - - | - kind: ClusterConfiguration - controllerManager: - extraArgs: - "v": "5" - "kube-api-qps": "5000" - "kube-api-burst": "10000" - extraVolumes: - - name: tmp-controller-manager - hostPath: /var/components/controller-manager/extras/tmp - mountPath: /extras/tmp - readOnly: false - pathType: DirectoryOrCreate - - | - kind: ClusterConfiguration - scheduler: - extraArgs: - "v": "5" - "kube-api-qps": "5000" - "kube-api-burst": "10000" - extraVolumes: - - name: tmp-scheduler - hostPath: /var/components/scheduler/extras/tmp - mountPath: /extras/tmp - readOnly: false - pathType: DirectoryOrCreate - # mount the local file on the control plane - extraMounts: - - hostPath: /workdir/clusters/ - containerPath: /etc/kwok/ - - hostPath: /workdir/clusters//manifests - containerPath: /etc/kubernetes/manifests - - hostPath: /workdir/clusters//pki - containerPath: /etc/kubernetes/pki - - hostPath: /extras/etcd - containerPath: /var/components/etcd/extras/tmp - readOnly: false - - hostPath: /extras/apiserver - containerPath: /var/components/apiserver/extras/tmp - readOnly: false - - hostPath: /extras/controller-manager - containerPath: /var/components/controller-manager/extras/tmp - readOnly: false - - hostPath: /extras/scheduler - containerPath: /var/components/scheduler/extras/tmp - readOnly: false - - hostPath: /extras/controller - containerPath: /var/components/controller/extras/tmp - readOnly: false - - hostPath: /extras/prometheus - containerPath: /var/components/prometheus/extras/tmp - readOnly: false - - hostPath: /workdir/clusters//prometheus.yaml - containerPath: /var/components/prometheus/etc/prometheus/prometheus.yaml - readOnly: false + role: control-plane EOF podman pull localhost/kwok:test cat </workdir/clusters//manifests/kwok-controller.yaml diff --git a/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt index a19d21aae3..25cf33db32 100644 --- a/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/kind-podman/create_cluster_with_verbosity.txt @@ -21,12 +21,83 @@ endpoint: 0.0.0.0:4317 samplingRatePerMillion: 1000000 EOF cat </workdir/clusters//kind.yaml -kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +kubeadmConfigPatches: +- | + apiServer: + extraArgs: + audit-log-path: /var/log/kubernetes/audit.log + audit-policy-file: /etc/kubernetes/audit/audit.yaml + enable-priority-and-fairness: "false" + max-mutating-requests-inflight: "0" + max-requests-inflight: "0" + tracing-config-file: /etc/kubernetes/apiserver-tracing-config.yaml + v: "4" + extraVolumes: + - hostPath: /workdir/clusters//audit.yaml + mountPath: /etc/kubernetes/audit/audit.yaml + name: audit-policy-file + pathType: File + readOnly: true + - hostPath: /workdir/clusters//logs/audit.log + mountPath: /var/log/kubernetes/audit.log + name: audit-log-path + pathType: File + - hostPath: /workdir/clusters//apiserver-tracing-config.yaml + mountPath: /etc/kubernetes/apiserver-tracing-config.yaml + name: apiserver-tracing-config + pathType: File + readOnly: true + apiVersion: kubeadm.k8s.io/v1beta3 + controllerManager: + extraArgs: + kube-api-burst: "10000" + kube-api-qps: "5000" + v: "4" + dns: {} + etcd: + local: + dataDir: /var/lib/etcd + extraArgs: + log-level: debug + kind: ClusterConfiguration + networking: {} + scheduler: + extraArgs: + config: /etc/kubernetes/scheduler/scheduler.yaml + kube-api-burst: "10000" + kube-api-qps: "5000" + v: "4" + extraVolumes: + - hostPath: /workdir/clusters//scheduler.yaml + mountPath: /etc/kubernetes/scheduler/scheduler.yaml + name: config + pathType: File + readOnly: true networking: apiServerPort: 32766 nodes: -- role: control-plane +- extraMounts: + - containerPath: /etc/kwok/ + hostPath: /workdir/clusters/ + - containerPath: /etc/kubernetes/manifests + hostPath: /workdir/clusters//manifests + - containerPath: /etc/kubernetes/pki + hostPath: /workdir/clusters//pki + - containerPath: /var/components/apiserver/etc/kubernetes/audit/audit.yaml + hostPath: /workdir/clusters//audit.yaml + readOnly: true + - containerPath: /var/components/apiserver/var/log/kubernetes/audit.log + hostPath: /workdir/clusters//logs/audit.log + - containerPath: /var/components/apiserver/etc/kubernetes/apiserver-tracing-config.yaml + hostPath: /workdir/clusters//apiserver-tracing-config.yaml + readOnly: true + - containerPath: /var/components/scheduler/etc/kubernetes/scheduler/scheduler.yaml + hostPath: /workdir/clusters//scheduler.yaml + readOnly: true + - containerPath: /var/components/prometheus/etc/prometheus/prometheus.yaml + hostPath: /workdir/clusters//prometheus.yaml extraPortMappings: - containerPort: 8080 hostPort: 8000 @@ -40,84 +111,7 @@ nodes: - containerPort: 2379 hostPort: 32765 protocol: TCP - kubeadmConfigPatches: - - | - kind: ClusterConfiguration - etcd: - local: - extraArgs: - "log-level": "debug" - - | - kind: ClusterConfiguration - apiServer: - extraArgs: - "audit-policy-file": "/etc/kubernetes/audit/audit.yaml" - "audit-log-path": "/var/log/kubernetes/audit.log" - "tracing-config-file": "/etc/kubernetes/apiserver-tracing-config.yaml" - "v": "4" - "max-requests-inflight": "0" - "max-mutating-requests-inflight": "0" - "enable-priority-and-fairness": "false" - extraVolumes: - - name: audit-policy-file - hostPath: /var/components/apiserver/etc/kubernetes/audit/audit.yaml - mountPath: /etc/kubernetes/audit/audit.yaml - readOnly: true - pathType: File - - name: audit-log-path - hostPath: /var/components/apiserver/var/log/kubernetes/audit.log - mountPath: /var/log/kubernetes/audit.log - readOnly: false - pathType: File - - name: apiserver-tracing-config - hostPath: /var/components/apiserver/etc/kubernetes/apiserver-tracing-config.yaml - mountPath: /etc/kubernetes/apiserver-tracing-config.yaml - readOnly: true - pathType: File - - | - kind: ClusterConfiguration - controllerManager: - extraArgs: - "v": "4" - "kube-api-qps": "5000" - "kube-api-burst": "10000" - - | - kind: ClusterConfiguration - scheduler: - extraArgs: - "config": "/etc/kubernetes/scheduler/scheduler.yaml" - "v": "4" - "kube-api-qps": "5000" - "kube-api-burst": "10000" - extraVolumes: - - name: config - hostPath: /var/components/scheduler/etc/kubernetes/scheduler/scheduler.yaml - mountPath: /etc/kubernetes/scheduler/scheduler.yaml - readOnly: true - pathType: File - # mount the local file on the control plane - extraMounts: - - hostPath: /workdir/clusters/ - containerPath: /etc/kwok/ - - hostPath: /workdir/clusters//manifests - containerPath: /etc/kubernetes/manifests - - hostPath: /workdir/clusters//pki - containerPath: /etc/kubernetes/pki - - hostPath: /workdir/clusters//audit.yaml - containerPath: /var/components/apiserver/etc/kubernetes/audit/audit.yaml - readOnly: true - - hostPath: /workdir/clusters//logs/audit.log - containerPath: /var/components/apiserver/var/log/kubernetes/audit.log - readOnly: false - - hostPath: /workdir/clusters//apiserver-tracing-config.yaml - containerPath: /var/components/apiserver/etc/kubernetes/apiserver-tracing-config.yaml - readOnly: true - - hostPath: /workdir/clusters//scheduler.yaml - containerPath: /var/components/scheduler/etc/kubernetes/scheduler/scheduler.yaml - readOnly: true - - hostPath: /workdir/clusters//prometheus.yaml - containerPath: /var/components/prometheus/etc/prometheus/prometheus.yaml - readOnly: false + role: control-plane EOF podman pull localhost/kwok:test cat </workdir/clusters//manifests/kwok-controller.yaml diff --git a/test/kwokctl/testdata/kind/create_cluster.txt b/test/kwokctl/testdata/kind/create_cluster.txt index 38f46c794d..ed97d6ca34 100644 --- a/test/kwokctl/testdata/kind/create_cluster.txt +++ b/test/kwokctl/testdata/kind/create_cluster.txt @@ -7,44 +7,45 @@ docker pull docker.io/kindest/node:v1.29.0 mkdir -p /workdir/clusters//pki mkdir -p /workdir/clusters//manifests cat </workdir/clusters//kind.yaml -kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +kubeadmConfigPatches: +- | + apiServer: + extraArgs: + enable-priority-and-fairness: "false" + max-mutating-requests-inflight: "0" + max-requests-inflight: "0" + apiVersion: kubeadm.k8s.io/v1beta3 + controllerManager: + extraArgs: + kube-api-burst: "10000" + kube-api-qps: "5000" + dns: {} + etcd: + local: + dataDir: /var/lib/etcd + kind: ClusterConfiguration + networking: {} + scheduler: + extraArgs: + kube-api-burst: "10000" + kube-api-qps: "5000" networking: apiServerPort: 32766 nodes: -- role: control-plane +- extraMounts: + - containerPath: /etc/kwok/ + hostPath: /workdir/clusters/ + - containerPath: /etc/kubernetes/manifests + hostPath: /workdir/clusters//manifests + - containerPath: /etc/kubernetes/pki + hostPath: /workdir/clusters//pki extraPortMappings: - containerPort: 2379 hostPort: 32765 protocol: TCP - kubeadmConfigPatches: - - | - kind: ClusterConfiguration - apiServer: - extraArgs: - "max-requests-inflight": "0" - "max-mutating-requests-inflight": "0" - "enable-priority-and-fairness": "false" - - | - kind: ClusterConfiguration - controllerManager: - extraArgs: - "kube-api-qps": "5000" - "kube-api-burst": "10000" - - | - kind: ClusterConfiguration - scheduler: - extraArgs: - "kube-api-qps": "5000" - "kube-api-burst": "10000" - # mount the local file on the control plane - extraMounts: - - hostPath: /workdir/clusters/ - containerPath: /etc/kwok/ - - hostPath: /workdir/clusters//manifests - containerPath: /etc/kubernetes/manifests - - hostPath: /workdir/clusters//pki - containerPath: /etc/kubernetes/pki + role: control-plane EOF docker pull localhost/kwok:test cat </workdir/clusters//manifests/kwok-controller.yaml diff --git a/test/kwokctl/testdata/kind/create_cluster_with_extra.txt b/test/kwokctl/testdata/kind/create_cluster_with_extra.txt index 4f361d43af..5e5e8e0060 100644 --- a/test/kwokctl/testdata/kind/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/kind/create_cluster_with_extra.txt @@ -7,12 +7,72 @@ docker pull docker.io/kindest/node:v1.29.0 mkdir -p /workdir/clusters//pki mkdir -p /workdir/clusters//manifests cat </workdir/clusters//kind.yaml -kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +kubeadmConfigPatches: +- | + apiServer: + extraArgs: + enable-priority-and-fairness: "false" + max-mutating-requests-inflight: "0" + max-requests-inflight: "0" + v: "5" + extraVolumes: + - hostPath: /extras/apiserver + mountPath: /extras/tmp + name: tmp-apiserver + pathType: DirectoryOrCreate + apiVersion: kubeadm.k8s.io/v1beta3 + controllerManager: + extraArgs: + kube-api-burst: "10000" + kube-api-qps: "5000" + v: "5" + extraVolumes: + - hostPath: /extras/controller-manager + mountPath: /extras/tmp + name: tmp-controller-manager + pathType: DirectoryOrCreate + dns: {} + etcd: + local: + dataDir: /var/lib/etcd + extraArgs: + log-level: debug + kind: ClusterConfiguration + networking: {} + scheduler: + extraArgs: + kube-api-burst: "10000" + kube-api-qps: "5000" + v: "5" + extraVolumes: + - hostPath: /extras/scheduler + mountPath: /extras/tmp + name: tmp-scheduler + pathType: DirectoryOrCreate networking: apiServerPort: 32766 nodes: -- role: control-plane +- extraMounts: + - containerPath: /etc/kwok/ + hostPath: /workdir/clusters/ + - containerPath: /etc/kubernetes/manifests + hostPath: /workdir/clusters//manifests + - containerPath: /etc/kubernetes/pki + hostPath: /workdir/clusters//pki + - containerPath: /var/components/apiserver/extras/tmp + hostPath: /extras/apiserver + - containerPath: /var/components/controller-manager/extras/tmp + hostPath: /extras/controller-manager + - containerPath: /var/components/scheduler/extras/tmp + hostPath: /extras/scheduler + - containerPath: /var/components/controller/extras/tmp + hostPath: /extras/controller + - containerPath: /var/components/prometheus/extras/tmp + hostPath: /extras/prometheus + - containerPath: /var/components/prometheus/etc/prometheus/prometheus.yaml + hostPath: /workdir/clusters//prometheus.yaml extraPortMappings: - containerPort: 9090 hostPort: 9090 @@ -20,88 +80,7 @@ nodes: - containerPort: 2379 hostPort: 32765 protocol: TCP - kubeadmConfigPatches: - - | - kind: ClusterConfiguration - etcd: - local: - extraArgs: - "log-level": "debug" - extraVolumes: - - name: tmp-etcd - hostPath: /var/components/etcd/extras/tmp - mountPath: /extras/tmp - readOnly: false - pathType: DirectoryOrCreate - - | - kind: ClusterConfiguration - apiServer: - extraArgs: - "v": "5" - "max-requests-inflight": "0" - "max-mutating-requests-inflight": "0" - "enable-priority-and-fairness": "false" - extraVolumes: - - name: tmp-apiserver - hostPath: /var/components/apiserver/extras/tmp - mountPath: /extras/tmp - readOnly: false - pathType: DirectoryOrCreate - - | - kind: ClusterConfiguration - controllerManager: - extraArgs: - "v": "5" - "kube-api-qps": "5000" - "kube-api-burst": "10000" - extraVolumes: - - name: tmp-controller-manager - hostPath: /var/components/controller-manager/extras/tmp - mountPath: /extras/tmp - readOnly: false - pathType: DirectoryOrCreate - - | - kind: ClusterConfiguration - scheduler: - extraArgs: - "v": "5" - "kube-api-qps": "5000" - "kube-api-burst": "10000" - extraVolumes: - - name: tmp-scheduler - hostPath: /var/components/scheduler/extras/tmp - mountPath: /extras/tmp - readOnly: false - pathType: DirectoryOrCreate - # mount the local file on the control plane - extraMounts: - - hostPath: /workdir/clusters/ - containerPath: /etc/kwok/ - - hostPath: /workdir/clusters//manifests - containerPath: /etc/kubernetes/manifests - - hostPath: /workdir/clusters//pki - containerPath: /etc/kubernetes/pki - - hostPath: /extras/etcd - containerPath: /var/components/etcd/extras/tmp - readOnly: false - - hostPath: /extras/apiserver - containerPath: /var/components/apiserver/extras/tmp - readOnly: false - - hostPath: /extras/controller-manager - containerPath: /var/components/controller-manager/extras/tmp - readOnly: false - - hostPath: /extras/scheduler - containerPath: /var/components/scheduler/extras/tmp - readOnly: false - - hostPath: /extras/controller - containerPath: /var/components/controller/extras/tmp - readOnly: false - - hostPath: /extras/prometheus - containerPath: /var/components/prometheus/extras/tmp - readOnly: false - - hostPath: /workdir/clusters//prometheus.yaml - containerPath: /var/components/prometheus/etc/prometheus/prometheus.yaml - readOnly: false + role: control-plane EOF docker pull localhost/kwok:test cat </workdir/clusters//manifests/kwok-controller.yaml diff --git a/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt b/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt index 2b129dbfcd..8c1f070abf 100644 --- a/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt +++ b/test/kwokctl/testdata/kind/create_cluster_with_verbosity.txt @@ -21,12 +21,83 @@ endpoint: 0.0.0.0:4317 samplingRatePerMillion: 1000000 EOF cat </workdir/clusters//kind.yaml -kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +kubeadmConfigPatches: +- | + apiServer: + extraArgs: + audit-log-path: /var/log/kubernetes/audit.log + audit-policy-file: /etc/kubernetes/audit/audit.yaml + enable-priority-and-fairness: "false" + max-mutating-requests-inflight: "0" + max-requests-inflight: "0" + tracing-config-file: /etc/kubernetes/apiserver-tracing-config.yaml + v: "4" + extraVolumes: + - hostPath: /workdir/clusters//audit.yaml + mountPath: /etc/kubernetes/audit/audit.yaml + name: audit-policy-file + pathType: File + readOnly: true + - hostPath: /workdir/clusters//logs/audit.log + mountPath: /var/log/kubernetes/audit.log + name: audit-log-path + pathType: File + - hostPath: /workdir/clusters//apiserver-tracing-config.yaml + mountPath: /etc/kubernetes/apiserver-tracing-config.yaml + name: apiserver-tracing-config + pathType: File + readOnly: true + apiVersion: kubeadm.k8s.io/v1beta3 + controllerManager: + extraArgs: + kube-api-burst: "10000" + kube-api-qps: "5000" + v: "4" + dns: {} + etcd: + local: + dataDir: /var/lib/etcd + extraArgs: + log-level: debug + kind: ClusterConfiguration + networking: {} + scheduler: + extraArgs: + config: /etc/kubernetes/scheduler/scheduler.yaml + kube-api-burst: "10000" + kube-api-qps: "5000" + v: "4" + extraVolumes: + - hostPath: /workdir/clusters//scheduler.yaml + mountPath: /etc/kubernetes/scheduler/scheduler.yaml + name: config + pathType: File + readOnly: true networking: apiServerPort: 32766 nodes: -- role: control-plane +- extraMounts: + - containerPath: /etc/kwok/ + hostPath: /workdir/clusters/ + - containerPath: /etc/kubernetes/manifests + hostPath: /workdir/clusters//manifests + - containerPath: /etc/kubernetes/pki + hostPath: /workdir/clusters//pki + - containerPath: /var/components/apiserver/etc/kubernetes/audit/audit.yaml + hostPath: /workdir/clusters//audit.yaml + readOnly: true + - containerPath: /var/components/apiserver/var/log/kubernetes/audit.log + hostPath: /workdir/clusters//logs/audit.log + - containerPath: /var/components/apiserver/etc/kubernetes/apiserver-tracing-config.yaml + hostPath: /workdir/clusters//apiserver-tracing-config.yaml + readOnly: true + - containerPath: /var/components/scheduler/etc/kubernetes/scheduler/scheduler.yaml + hostPath: /workdir/clusters//scheduler.yaml + readOnly: true + - containerPath: /var/components/prometheus/etc/prometheus/prometheus.yaml + hostPath: /workdir/clusters//prometheus.yaml extraPortMappings: - containerPort: 8080 hostPort: 8000 @@ -40,84 +111,7 @@ nodes: - containerPort: 2379 hostPort: 32765 protocol: TCP - kubeadmConfigPatches: - - | - kind: ClusterConfiguration - etcd: - local: - extraArgs: - "log-level": "debug" - - | - kind: ClusterConfiguration - apiServer: - extraArgs: - "audit-policy-file": "/etc/kubernetes/audit/audit.yaml" - "audit-log-path": "/var/log/kubernetes/audit.log" - "tracing-config-file": "/etc/kubernetes/apiserver-tracing-config.yaml" - "v": "4" - "max-requests-inflight": "0" - "max-mutating-requests-inflight": "0" - "enable-priority-and-fairness": "false" - extraVolumes: - - name: audit-policy-file - hostPath: /var/components/apiserver/etc/kubernetes/audit/audit.yaml - mountPath: /etc/kubernetes/audit/audit.yaml - readOnly: true - pathType: File - - name: audit-log-path - hostPath: /var/components/apiserver/var/log/kubernetes/audit.log - mountPath: /var/log/kubernetes/audit.log - readOnly: false - pathType: File - - name: apiserver-tracing-config - hostPath: /var/components/apiserver/etc/kubernetes/apiserver-tracing-config.yaml - mountPath: /etc/kubernetes/apiserver-tracing-config.yaml - readOnly: true - pathType: File - - | - kind: ClusterConfiguration - controllerManager: - extraArgs: - "v": "4" - "kube-api-qps": "5000" - "kube-api-burst": "10000" - - | - kind: ClusterConfiguration - scheduler: - extraArgs: - "config": "/etc/kubernetes/scheduler/scheduler.yaml" - "v": "4" - "kube-api-qps": "5000" - "kube-api-burst": "10000" - extraVolumes: - - name: config - hostPath: /var/components/scheduler/etc/kubernetes/scheduler/scheduler.yaml - mountPath: /etc/kubernetes/scheduler/scheduler.yaml - readOnly: true - pathType: File - # mount the local file on the control plane - extraMounts: - - hostPath: /workdir/clusters/ - containerPath: /etc/kwok/ - - hostPath: /workdir/clusters//manifests - containerPath: /etc/kubernetes/manifests - - hostPath: /workdir/clusters//pki - containerPath: /etc/kubernetes/pki - - hostPath: /workdir/clusters//audit.yaml - containerPath: /var/components/apiserver/etc/kubernetes/audit/audit.yaml - readOnly: true - - hostPath: /workdir/clusters//logs/audit.log - containerPath: /var/components/apiserver/var/log/kubernetes/audit.log - readOnly: false - - hostPath: /workdir/clusters//apiserver-tracing-config.yaml - containerPath: /var/components/apiserver/etc/kubernetes/apiserver-tracing-config.yaml - readOnly: true - - hostPath: /workdir/clusters//scheduler.yaml - containerPath: /var/components/scheduler/etc/kubernetes/scheduler/scheduler.yaml - readOnly: true - - hostPath: /workdir/clusters//prometheus.yaml - containerPath: /var/components/prometheus/etc/prometheus/prometheus.yaml - readOnly: false + role: control-plane EOF docker pull localhost/kwok:test cat </workdir/clusters//manifests/kwok-controller.yaml diff --git a/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt b/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt index d5b377e805..421dd3a3a4 100644 --- a/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/nerdctl/create_cluster_with_extra.txt @@ -139,7 +139,7 @@ users: EOF # Save cluster config to /workdir/clusters//kwok.yaml nerdctl network create kwok- --label=com.docker.compose.project=kwok- -nerdctl create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp --volume=/extras/etcd:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug +nerdctl create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug nerdctl create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/extras/apiserver:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.29.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt --v=5 nerdctl create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/extras/controller-manager:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.29.0 --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=5 nerdctl create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --restart=unless-stopped --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/extras/scheduler:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.29.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=5 diff --git a/test/kwokctl/testdata/podman/create_cluster_with_extra.txt b/test/kwokctl/testdata/podman/create_cluster_with_extra.txt index 7b6c719a02..d91c07b630 100644 --- a/test/kwokctl/testdata/podman/create_cluster_with_extra.txt +++ b/test/kwokctl/testdata/podman/create_cluster_with_extra.txt @@ -139,7 +139,7 @@ users: EOF # Save cluster config to /workdir/clusters//kwok.yaml podman network create kwok- --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- -podman create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp --volume=/extras/etcd:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug +podman create --name=kwok--etcd --pull=never --entrypoint=etcd --network=kwok- --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32765:2379/tcp --env=TEST_KEY=TEST_VALUE registry.k8s.io/etcd:3.5.11-0 --name=node0 --auto-compaction-retention=1 --quota-backend-bytes=8589934592 --data-dir=/etcd-data --initial-advertise-peer-urls=http://0.0.0.0:2380 --listen-peer-urls=http://0.0.0.0:2380 --advertise-client-urls=http://0.0.0.0:2379 --listen-client-urls=http://0.0.0.0:2379 --initial-cluster=node0=http://0.0.0.0:2380 --log-level=debug podman create --name=kwok--kube-apiserver --pull=never --entrypoint=kube-apiserver --network=kwok- --requires=kwok--etcd --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --publish=32766:6443/tcp --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/extras/apiserver:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-apiserver:v1.29.0 --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://kwok--etcd:2379 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=6443 --tls-cert-file=/etc/kubernetes/pki/admin.crt --tls-private-key-file=/etc/kubernetes/pki/admin.key --client-ca-file=/etc/kubernetes/pki/ca.crt --service-account-key-file=/etc/kubernetes/pki/admin.key --service-account-signing-key-file=/etc/kubernetes/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=/etc/kubernetes/pki/admin.key --proxy-client-cert-file=/etc/kubernetes/pki/admin.crt --v=5 podman create --name=kwok--kube-controller-manager --pull=never --entrypoint=kube-controller-manager --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/extras/controller-manager:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-controller-manager:v1.29.0 --node-monitor-period=25s --node-monitor-grace-period=3m20s --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10257 --root-ca-file=/etc/kubernetes/pki/ca.crt --service-account-private-key-file=/etc/kubernetes/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=5 podman create --name=kwok--kube-scheduler --pull=never --entrypoint=kube-scheduler --network=kwok- --requires=kwok--kube-apiserver --restart=unless-stopped --label=io.podman.compose.project=kwok- --label=com.docker.compose.project=kwok- --volume=/workdir/clusters//kubeconfig:~/.kube/config:ro --volume=/workdir/clusters//pki/ca.crt:/etc/kubernetes/pki/ca.crt:ro --volume=/workdir/clusters//pki/admin.crt:/etc/kubernetes/pki/admin.crt:ro --volume=/workdir/clusters//pki/admin.key:/etc/kubernetes/pki/admin.key:ro --volume=/extras/scheduler:/extras/tmp --env=TEST_KEY=TEST_VALUE registry.k8s.io/kube-scheduler:v1.29.0 --kubeconfig=~/.kube/config --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=10259 --kube-api-qps=5000 --kube-api-burst=10000 --v=5