[kwokctl] Integrate metrics-server

kustomize/kwokctl/resource/node.yaml

@@ -20,6 +20,7 @@ template: |-
     annotations:
       kwok.x-k8s.io/node: fake
       node.alpha.kubernetes.io/ttl: "0"
+      metrics.k8s.io/resource-metrics-path: "/metrics/nodes/{{ Name }}/metrics/resource"
     labels:
       beta.kubernetes.io/arch: {{ .nodeInfo.architecture }}
       beta.kubernetes.io/os: {{ .nodeInfo.operatingSystem }}

pkg/apis/config/v1alpha1/kwokctl_configuration_types.go

@@ -128,6 +128,9 @@ type KwokctlConfigurationOptions struct {
 	// is the default value for env KWOK_JAEGER_VERSION
 	JaegerVersion string `json:"jaegerVersion,omitempty"`
 
+	// MetricsServerVersion is the version of metrics-server to use.
+	MetricsServerVersion string `json:"metricsServerVersion,omitempty"`
+
 	// DockerComposeVersion is the version of docker-compose to use.
 	// is the default value for env KWOK_DOCKER_COMPOSE_VERSION
 	// Deprecated: docker compose will be removed in a future release
@@ -161,6 +164,10 @@ type KwokctlConfigurationOptions struct {
 	// +default=false
 	DisableKubeControllerManager *bool `json:"disableKubeControllerManager,omitempty"`
 
+	// EnableMetricsServer is the flag to enable metrics-server.
+	// +default=false
+	EnableMetricsServer *bool `json:"enableMetricsServer,omitempty"`
+
 	// KubeImagePrefix is the prefix of the kubernetes image.
 	// is the default value for env KWOK_KUBE_IMAGE_PREFIX
 	//+k8s:conversion-gen=false
@@ -190,6 +197,10 @@ type KwokctlConfigurationOptions struct {
 	//+k8s:conversion-gen=false
 	JaegerImagePrefix string `json:"jaegerImagePrefix,omitempty"`
 
+	// MetricsServerImagePrefix is the prefix of the metrics-server image.
+	//+k8s:conversion-gen=false
+	MetricsServerImagePrefix string `json:"metricsServerImagePrefix,omitempty"`
+
 	// EtcdImage is the image of etcd.
 	// is the default value for flag --etcd-image and env KWOK_ETCD_IMAGE
 	EtcdImage string `json:"etcdImage,omitempty"`
@@ -221,6 +232,9 @@ type KwokctlConfigurationOptions struct {
 	// is the default value for flag --jaeger-image and env KWOK_JAEGER_IMAGE
 	JaegerImage string `json:"jaegerImage,omitempty"`
 
+	// MetricsServerImage is the image of metrics-server.
+	MetricsServerImage string `json:"metricsServerImage,omitempty"`
+
 	// KindNodeImagePrefix is the prefix of the kind node image.
 	// is the default value for env KWOK_KIND_NODE_IMAGE_PREFIX
 	//+k8s:conversion-gen=false
@@ -320,6 +334,13 @@ type KwokctlConfigurationOptions struct {
 	//+k8s:conversion-gen=false
 	JaegerBinaryTar string `json:"jaegerBinaryTar,omitempty"`
 
+	// MetricsServerBinaryPrefix is the prefix of the metrics-server binary.
+	//+k8s:conversion-gen=false
+	MetricsServerBinaryPrefix string `json:"metricsServerBinaryPrefix,omitempty"`
+
+	// MetricsServerBinary is the binary of metrics-server.
+	MetricsServerBinary string `json:"metricsServerBinary,omitempty"`
+
 	// DockerComposeBinaryPrefix is the binary of docker-compose.
 	// is the default value for env KWOK_DOCKER_COMPOSE_BINARY_PREFIX
 	// Deprecated: docker compose will be removed in a future release
@@ -387,6 +408,9 @@ type KwokctlConfigurationOptions struct {
 	// is the default value for flag --controller-port and env KWOK_CONTROLLER_PORT
 	KwokControllerPort uint32 `json:"kwokControllerPort,omitempty"`
 
+	// MetricsServerPort is metrics-server port that is exposed to the host.
+	MetricsServerPort uint32 `json:"metricsServerPort,omitempty"`
+
 	// CacheDir is the directory of the cache.
 	CacheDir string `json:"cacheDir,omitempty"`
 

pkg/apis/internalversion/kwokctl_configuration_types.go

@@ -104,6 +104,9 @@ type KwokctlConfigurationOptions struct {
 	// JaegerVersion is the version of Jaeger to use.
 	JaegerVersion string
 
+	// MetricsServerVersion is the version of metrics-server to use.
+	MetricsServerVersion string
+
 	// DockerComposeVersion is the version of docker-compose to use.
 	DockerComposeVersion string
 
@@ -126,6 +129,9 @@ type KwokctlConfigurationOptions struct {
 	// DisableKubeControllerManager is the flag to disable kube-controller-manager.
 	DisableKubeControllerManager bool
 
+	// EnableMetricsServer is the flag to enable metrics-server.
+	EnableMetricsServer bool
+
 	// EtcdImage is the image of etcd.
 	EtcdImage string
 
@@ -150,6 +156,9 @@ type KwokctlConfigurationOptions struct {
 	// JaegerImage is the image of Jaeger
 	JaegerImage string
 
+	// MetricsServerImage is the image of metrics-server.
+	MetricsServerImage string
+
 	// KindNodeImage is the image of kind node.
 	KindNodeImage string
 
@@ -200,6 +209,9 @@ type KwokctlConfigurationOptions struct {
 	// Deprecated: Use JaegerBinary instead
 	JaegerBinaryTar string
 
+	// MetricsServerBinary is the binary of metrics-server.
+	MetricsServerBinary string
+
 	// DockerComposeBinary is the binary of Docker compose.
 	DockerComposeBinary string
 
@@ -239,6 +251,9 @@ type KwokctlConfigurationOptions struct {
 	// KwokControllerPort is kwok-controller port that is exposed to the host.
 	KwokControllerPort uint32
 
+	// MetricsServerPort is metrics-server port that is exposed to the host.
+	MetricsServerPort uint32
+
 	// CacheDir is the directory of the cache.
 	CacheDir string
 

pkg/config/vars.go

@@ -246,6 +246,8 @@ func setKwokctlConfigurationDefaults(config *configv1alpha1.KwokctlConfiguration
 
 	setKwokctlJaegerConfig(conf)
 
+	setMetricsServerConfig(conf)
+
 	return config
 }
 
@@ -543,6 +545,34 @@ func setKwokctlJaegerConfig(conf *configv1alpha1.KwokctlConfigurationOptions) {
 	}
 }
 
+func setMetricsServerConfig(conf *configv1alpha1.KwokctlConfigurationOptions) {
+	if conf.MetricsServerVersion == "" {
+		conf.MetricsServerVersion = consts.MetricsServerVersion
+	}
+	conf.MetricsServerVersion = version.AddPrefixV(envs.GetEnvWithPrefix("METRICS_SERVER_VERSION", conf.MetricsServerVersion))
+
+	if conf.MetricsServerImagePrefix == "" {
+		conf.MetricsServerImagePrefix = consts.MetricsServerImagePrefix
+	}
+	conf.MetricsServerImagePrefix = envs.GetEnvWithPrefix("METRICS_SERVER_IMAGE_PREFIX", conf.MetricsServerImagePrefix)
+
+	if conf.MetricsServerImage == "" {
+		conf.MetricsServerImage = joinImageURI(conf.MetricsServerImagePrefix, "metrics-server", version.AddPrefixV(conf.MetricsServerVersion))
+	}
+	conf.MetricsServerImage = envs.GetEnvWithPrefix("METRICS_SERVER_IMAGE", conf.MetricsServerImage)
+
+	if conf.MetricsServerBinaryPrefix == "" {
+		conf.MetricsServerBinaryPrefix = consts.MetricsServerBinaryPrefix + "/" + conf.MetricsServerVersion
+	}
+	conf.MetricsServerBinaryPrefix = envs.GetEnvWithPrefix("METRICS_SERVER_BINARY_PREFIX", conf.MetricsServerBinaryPrefix)
+
+	if conf.MetricsServerBinaryPrefix != "" &&
+		conf.MetricsServerBinary == "" {
+		conf.MetricsServerBinary = conf.MetricsServerBinaryPrefix + "/metrics-server-" + GOOS + "-" + GOARCH + conf.BinSuffix
+	}
+	conf.MetricsServerBinary = envs.GetEnvWithPrefix("METRICS_SERVER_BINARY", conf.MetricsServerBinary)
+}
+
 // joinImageURI joins the image URI.
 func joinImageURI(prefix, name, version string) string {
 	return prefix + "/" + name + ":" + version

pkg/consts/consts.go

@@ -61,6 +61,10 @@ var (
 	JaegerBinaryPrefix = "https://github.com/jaegertracing/jaeger/releases/download"
 	JaegerImagePrefix  = "docker.io/jaegertracing"
 
+	MetricsServerVersion      = "0.7.0"
+	MetricsServerBinaryPrefix = "https://github.com/kubernetes-sigs/metrics-server/releases/download"
+	MetricsServerImagePrefix  = "registry.k8s.io/metrics-server"
+
 	DefaultUnlimitedQPS   = 5000.0
 	DefaultUnlimitedBurst = 10000
 )
@@ -85,4 +89,5 @@ const (
 	ComponentDashboard             = "dashboard"
 	ComponentPrometheus            = "prometheus"
 	ComponentJaeger                = "jaeger"
+	ComponentMetricsServer         = "metrics-server"
 )

pkg/kwokctl/cmd/create/cluster/cluster.go

@@ -68,6 +68,7 @@ func NewCommand(ctx context.Context) *cobra.Command {
 	cmd.Flags().StringVar(&flags.Options.KubeSchedulerConfig, "kube-scheduler-config", flags.Options.KubeSchedulerConfig, `Path to a kube-scheduler configuration file`)
 	cmd.Flags().BoolVar(&flags.Options.DisableKubeScheduler, "disable-kube-scheduler", flags.Options.DisableKubeScheduler, `Disable the kube-scheduler`)
 	cmd.Flags().BoolVar(&flags.Options.DisableKubeControllerManager, "disable-kube-controller-manager", flags.Options.DisableKubeControllerManager, `Disable the kube-controller-manager`)
+	cmd.Flags().BoolVar(&flags.Options.EnableMetricsServer, "enable-metrics-server", flags.Options.EnableMetricsServer, `Enable the metrics-server`)
 	cmd.Flags().StringVar(&flags.Options.EtcdImage, "etcd-image", flags.Options.EtcdImage, `Image of etcd, only for docker/podman/nerdctl runtime
 '${KWOK_KUBE_IMAGE_PREFIX}/etcd:${KWOK_ETCD_VERSION}'
 `)
@@ -85,6 +86,9 @@ func NewCommand(ctx context.Context) *cobra.Command {
 	cmd.Flags().Uint32Var(&flags.Options.KubeSchedulerPort, "kube-scheduler-port", flags.Options.KubeSchedulerPort, `Port of kube-scheduler given to the host, only for binary and docker/podman/nerdctl runtime`)
 	cmd.Flags().StringVar(&flags.Options.KwokControllerImage, "kwok-controller-image", flags.Options.KwokControllerImage, `Image of kwok-controller, only for docker/podman/nerdctl/kind/kind-podman runtime
 '${KWOK_IMAGE_PREFIX}/kwok:${KWOK_VERSION}'
+`)
+	cmd.Flags().StringVar(&flags.Options.MetricsServerImage, "metrics-server-image", flags.Options.MetricsServerImage, `Image of metrics-server, only for docker/podman/nerdctl/kind/kind-podman runtime
+'${KWOK_METRICS_SERVER_IMAGE_PREFIX}/metrics-server:${KWOK_METRICS_SERVER_VERSION}'
 `)
 	cmd.Flags().StringVar(&flags.Options.PrometheusImage, "prometheus-image", flags.Options.PrometheusImage, `Image of Prometheus, only for docker/podman/nerdctl/kind/kind-podman runtime
 '${KWOK_PROMETHEUS_IMAGE_PREFIX}/prometheus:${KWOK_PROMETHEUS_VERSION}'
@@ -112,6 +116,7 @@ func NewCommand(ctx context.Context) *cobra.Command {
 	cmd.Flags().StringVar(&flags.Options.EtcdBinaryTar, "etcd-binary-tar", flags.Options.EtcdBinaryTar, `Tar of etcd, if --etcd-binary is set, this is ignored, only for binary runtime
 `)
 	_ = cmd.Flags().MarkDeprecated("etcd-binary-tar", "--etcd-binary-tar will be removed in a future release, please use --etcd-binary instead")
+	cmd.Flags().StringVar(&flags.Options.MetricsServerBinary, "metrics-server-binary", flags.Options.MetricsServerBinary, `Binary of metrics-server, only for binary runtime`)
 	cmd.Flags().StringVar(&flags.Options.PrometheusBinary, "prometheus-binary", flags.Options.PrometheusBinary, `Binary of Prometheus, only for binary runtime`)
 	cmd.Flags().StringVar(&flags.Options.PrometheusBinaryTar, "prometheus-binary-tar", flags.Options.PrometheusBinaryTar, `Tar of Prometheus, if --prometheus-binary is set, this is ignored, only for binary runtime
 `)
@@ -287,6 +292,10 @@ func runE(ctx context.Context, flags *flagpole) error {
 	if err != nil {
 		return fmt.Errorf("failed to init crds %q: %w", name, err)
 	}
+	err = rt.InitCRs(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to init crs %q: %w", name, err)
+	}
 
 	// Wait for cluster to be ready
 	if flags.Wait > 0 {

pkg/kwokctl/cmd/logs/logs.go

@@ -41,7 +41,7 @@ func NewCommand(ctx context.Context) *cobra.Command {
 
 	cmd := &cobra.Command{
 		Use:   "logs [command]",
-		Short: "Logs one of [audit, etcd, kube-apiserver, kube-controller-manager, kube-scheduler, kwok-controller, dashboard, prometheus, jaeger]",
+		Short: "Logs one of [audit, etcd, kube-apiserver, kube-controller-manager, kube-scheduler, kwok-controller, dashboard, metrics-server, prometheus, jaeger]",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if len(args) == 0 {
 				return cmd.Help()

pkg/kwokctl/components/kube_apiserver.go

@@ -170,6 +170,8 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon
 				"--service-account-key-file=/etc/kubernetes/pki/admin.key",
 				"--service-account-signing-key-file=/etc/kubernetes/pki/admin.key",
 				"--service-account-issuer=https://kubernetes.default.svc.cluster.local",
+				"--proxy-client-key-file=/etc/kubernetes/pki/admin.key",
+				"--proxy-client-cert-file=/etc/kubernetes/pki/admin.crt",
 			)
 			metric = &internalversion.ComponentMetric{
 				Scheme:             "https",
@@ -189,6 +191,8 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon
 				"--service-account-key-file="+conf.AdminKeyPath,
 				"--service-account-signing-key-file="+conf.AdminKeyPath,
 				"--service-account-issuer=https://kubernetes.default.svc.cluster.local",
+				"--proxy-client-key-file="+conf.AdminKeyPath,
+				"--proxy-client-cert-file="+conf.AdminCertPath,
 			)
 			metric = &internalversion.ComponentMetric{
 				Scheme:             "https",