kwokctl: add dashboard-metrics-scraper when have dashboard and metric…

…s-server

pkg/apis/config/v1alpha1/kwokctl_configuration_types.go

@@ -120,6 +120,9 @@ type KwokctlConfigurationOptions struct {
 	// DashboardVersion is the version of Kubernetes dashboard to use.
 	DashboardVersion string `json:"dashboardVersion,omitempty"`
 
+	// DashboardMetricsScraperVersion is the version of Kubernetes dashboard metrics scraper to use.
+	DashboardMetricsScraperVersion string `json:"dashboardMetricsScraperVersion,omitempty"`
+
 	// PrometheusVersion is the version of Prometheus to use.
 	// is the default value for env KWOK_PROMETHEUS_VERSION
 	PrometheusVersion string `json:"prometheusVersion,omitempty"`
@@ -224,6 +227,9 @@ type KwokctlConfigurationOptions struct {
 	// DashboardImage is the image of dashboard.
 	DashboardImage string `json:"dashboardImage,omitempty"`
 
+	// DashboardMetricsScraperImage is the image of dashboard metrics scraper.
+	DashboardMetricsScraperImage string `json:"dashboardMetricsScraperImage,omitempty"`
+
 	// PrometheusImage is the image of Prometheus.
 	// is the default value for flag --prometheus-image and env KWOK_PROMETHEUS_IMAGE
 	PrometheusImage string `json:"prometheusImage,omitempty"`

pkg/apis/internalversion/kwokctl_configuration_types.go

@@ -98,6 +98,9 @@ type KwokctlConfigurationOptions struct {
 	// DashboardVersion is the version of Kubernetes dashboard to use.
 	DashboardVersion string
 
+	// DashboardMetricsScraperVersion is the version of dashboard metrics scraper to use.
+	DashboardMetricsScraperVersion string
+
 	// PrometheusVersion is the version of Prometheus to use.
 	PrometheusVersion string
 
@@ -150,6 +153,9 @@ type KwokctlConfigurationOptions struct {
 	// DashboardImage is the image of dashboard.
 	DashboardImage string
 
+	// DashboardMetricsScraperImage is the image of dashboard metrics scraper.
+	DashboardMetricsScraperImage string
+
 	// PrometheusImage is the image of Prometheus.
 	PrometheusImage string
 

pkg/config/vars.go

@@ -453,6 +453,16 @@ func setKwokctlDashboardConfig(conf *configv1alpha1.KwokctlConfigurationOptions)
 	}
 	conf.DashboardImage = envs.GetEnvWithPrefix("DASHBOARD_IMAGE", conf.DashboardImage)
 
+	if conf.DashboardMetricsScraperVersion == "" {
+		conf.DashboardMetricsScraperVersion = consts.DashboardMetricsScraperVersion
+	}
+	conf.DashboardMetricsScraperVersion = version.AddPrefixV(envs.GetEnvWithPrefix("DASHBOARD_METRICS_SCRAPER_VERSION", conf.DashboardMetricsScraperVersion))
+
+	if conf.DashboardMetricsScraperImage == "" {
+		conf.DashboardMetricsScraperImage = joinImageURI(conf.DashboardImagePrefix, "metrics-scraper", conf.DashboardMetricsScraperVersion)
+	}
+	conf.DashboardMetricsScraperImage = envs.GetEnvWithPrefix("DASHBOARD_METRICS_SCRAPER_IMAGE", conf.DashboardMetricsScraperImage)
+
 	// TODO: Add dashboard binary
 	// if conf.DashboardBinaryPrefix == "" {
 	// 	conf.DashboardBinaryPrefix = consts.DashboardBinaryPrefix + "/" + conf.DashboardVersion

pkg/consts/consts.go

@@ -53,6 +53,8 @@ var (
 	DashboardBinaryPrefix = ""
 	DashboardImagePrefix  = "docker.io/kubernetesui"
 
+	DashboardMetricsScraperVersion = "1.0.9"
+
 	PrometheusVersion      = "2.49.1"
 	PrometheusBinaryPrefix = "https://github.com/prometheus/prometheus/releases/download"
 	PrometheusImagePrefix  = "docker.io/prom"
@@ -81,13 +83,14 @@ const (
 
 // The following components is provided.
 const (
-	ComponentEtcd                  = "etcd"
-	ComponentKubeApiserver         = "kube-apiserver"
-	ComponentKubeControllerManager = "kube-controller-manager"
-	ComponentKubeScheduler         = "kube-scheduler"
-	ComponentKwokController        = "kwok-controller"
-	ComponentDashboard             = "dashboard"
-	ComponentPrometheus            = "prometheus"
-	ComponentJaeger                = "jaeger"
-	ComponentMetricsServer         = "metrics-server"
+	ComponentEtcd                    = "etcd"
+	ComponentKubeApiserver           = "kube-apiserver"
+	ComponentKubeControllerManager   = "kube-controller-manager"
+	ComponentKubeScheduler           = "kube-scheduler"
+	ComponentKwokController          = "kwok-controller"
+	ComponentDashboard               = "dashboard"
+	ComponentDashboardMetricsScraper = "dashboard-metrics-scraper"
+	ComponentPrometheus              = "prometheus"
+	ComponentJaeger                  = "jaeger"
+	ComponentMetricsServer           = "metrics-server"
 )

pkg/kwokctl/components/dashboard.go

@@ -26,6 +26,7 @@ import (
 // BuildDashboardComponentConfig is the configuration for building the dashboard component.
 type BuildDashboardComponentConfig struct {
 	Runtime     string
+	ProjectName string
 	Binary      string
 	Image       string
 	Version     version.Version
@@ -35,6 +36,8 @@ type BuildDashboardComponentConfig struct {
 
 	Banner string
 
+	EnableMetrics bool
+
 	CaCertPath     string
 	AdminCertPath  string
 	AdminKeyPath   string
@@ -50,8 +53,18 @@ func BuildDashboardComponent(conf BuildDashboardComponentConfig) (component inte
 		"--enable-insecure-login",
 		"--enable-skip-login",
 		"--disable-settings-authorizer",
-		"--metrics-provider=none",
 	}
+	if conf.EnableMetrics {
+		switch GetRuntimeMode(conf.Runtime) {
+		case RuntimeModeContainer:
+			dashboardArgs = append(dashboardArgs, "--sidecar-host="+conf.ProjectName+"-"+consts.ComponentDashboardMetricsScraper+":8000")
+		default:
+			dashboardArgs = append(dashboardArgs, "--sidecar-host=127.0.0.1:8000")
+		}
+	} else {
+		dashboardArgs = append(dashboardArgs, "--metrics-provider=none")
+	}
+
 	if conf.Banner != "" {
 		dashboardArgs = append(dashboardArgs, "--system-banner="+conf.Banner)
 	}
@@ -62,7 +75,7 @@ func BuildDashboardComponent(conf BuildDashboardComponentConfig) (component inte
 	if GetRuntimeMode(conf.Runtime) != RuntimeModeNative {
 		dashboardArgs = append(dashboardArgs,
 			"--kubeconfig=/root/.kube/config",
-			"--insecure-port=8000",
+			"--insecure-port=8080",
 		)
 		volumes = append(volumes,
 			internalversion.Volume{
@@ -89,7 +102,7 @@ func BuildDashboardComponent(conf BuildDashboardComponentConfig) (component inte
 		ports = append(ports,
 			internalversion.Port{
 				Name:     "http",
-				Port:     8000,
+				Port:     8080,
 				HostPort: conf.Port,
 				Protocol: internalversion.ProtocolTCP,
 			},

pkg/kwokctl/components/dashboard_metrics_scraper.go

@@ -0,0 +1,94 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package components
+
+import (
+	"sigs.k8s.io/kwok/pkg/apis/internalversion"
+	"sigs.k8s.io/kwok/pkg/consts"
+	"sigs.k8s.io/kwok/pkg/utils/version"
+)
+
+// BuildDashboardMetricsScraperComponentConfig is the configuration for building the dashboard component.
+type BuildDashboardMetricsScraperComponentConfig struct {
+	Runtime string
+	Binary  string
+	Image   string
+	Version version.Version
+	Workdir string
+
+	CaCertPath     string
+	AdminCertPath  string
+	AdminKeyPath   string
+	KubeconfigPath string
+}
+
+// BuildDashboardMetricsScraperComponent builds the dashboard component.
+func BuildDashboardMetricsScraperComponent(conf BuildDashboardMetricsScraperComponentConfig) (component internalversion.Component, err error) {
+	dashboardArgs := []string{
+		"--db-file=/metrics.db",
+	}
+
+	user := ""
+	var volumes []internalversion.Volume
+	var ports []internalversion.Port
+	if GetRuntimeMode(conf.Runtime) != RuntimeModeNative {
+		dashboardArgs = append(dashboardArgs,
+			"--kubeconfig=/root/.kube/config",
+		)
+		volumes = append(volumes,
+			internalversion.Volume{
+				HostPath:  conf.KubeconfigPath,
+				MountPath: "/root/.kube/config",
+				ReadOnly:  true,
+			},
+			internalversion.Volume{
+				HostPath:  conf.CaCertPath,
+				MountPath: "/etc/kubernetes/pki/ca.crt",
+				ReadOnly:  true,
+			},
+			internalversion.Volume{
+				HostPath:  conf.AdminCertPath,
+				MountPath: "/etc/kubernetes/pki/admin.crt",
+				ReadOnly:  true,
+			},
+			internalversion.Volume{
+				HostPath:  conf.AdminKeyPath,
+				MountPath: "/etc/kubernetes/pki/admin.key",
+				ReadOnly:  true,
+			},
+		)
+		user = "root"
+	} else {
+		dashboardArgs = append(dashboardArgs,
+			"--kubeconfig="+conf.KubeconfigPath,
+		)
+	}
+
+	component = internalversion.Component{
+		Name:  consts.ComponentDashboardMetricsScraper,
+		Image: conf.Image,
+		Links: []string{
+			consts.ComponentMetricsServer,
+		},
+		WorkDir: conf.Workdir,
+		Ports:   ports,
+		Volumes: volumes,
+		Args:    dashboardArgs,
+		User:    user,
+	}
+	return component, nil
+}

pkg/kwokctl/runtime/compose/cluster.go

@@ -707,6 +707,7 @@ func (c *Cluster) addDashboard(_ context.Context, env *env) (err error) {
 	if conf.DashboardPort != 0 {
 		dashboardComponent, err := components.BuildDashboardComponent(components.BuildDashboardComponentConfig{
 			Runtime:        conf.Runtime,
+			ProjectName:    c.Name(),
 			Workdir:        env.workdir,
 			Image:          conf.DashboardImage,
 			BindAddress:    net.PublicAddress,
@@ -716,11 +717,28 @@ func (c *Cluster) addDashboard(_ context.Context, env *env) (err error) {
 			AdminKeyPath:   env.adminKeyPath,
 			Port:           conf.DashboardPort,
 			Banner:         fmt.Sprintf("Welcome to %s", c.Name()),
+			EnableMetrics:  conf.EnableMetricsServer,
 		})
 		if err != nil {
 			return err
 		}
 		env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, dashboardComponent)
+
+		if conf.EnableMetricsServer {
+			dashboardMetricsScraperComponent, err := components.BuildDashboardMetricsScraperComponent(components.BuildDashboardMetricsScraperComponentConfig{
+				Runtime:        conf.Runtime,
+				Workdir:        env.workdir,
+				Image:          conf.DashboardMetricsScraperImage,
+				KubeconfigPath: env.inClusterOnHostKubeconfigPath,
+				CaCertPath:     env.caCertPath,
+				AdminCertPath:  env.adminCertPath,
+				AdminKeyPath:   env.adminKeyPath,
+			})
+			if err != nil {
+				return err
+			}
+			env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, dashboardMetricsScraperComponent)
+		}
 	}
 	return nil
 }

pkg/kwokctl/runtime/kind/cluster.go

@@ -582,8 +582,9 @@ func (c *Cluster) addDashboard(ctx context.Context, env *env) (err error) {
 			CaCertPath:     env.caCertPath,
 			AdminCertPath:  env.adminCertPath,
 			AdminKeyPath:   env.adminKeyPath,
-			Port:           8000,
+			Port:           8080,
 			Banner:         fmt.Sprintf("Welcome to %s", c.Name()),
+			EnableMetrics:  conf.EnableMetricsServer,
 		})
 		if err != nil {
 			return fmt.Errorf("failed to build dashboard component: %w", err)
@@ -600,6 +601,30 @@ func (c *Cluster) addDashboard(ctx context.Context, env *env) (err error) {
 			return fmt.Errorf("failed to write: %w", err)
 		}
 		env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, dashboardComponent)
+
+		if conf.EnableMetricsServer {
+			dashboardMetricsScraperComponent, err := components.BuildDashboardMetricsScraperComponent(components.BuildDashboardMetricsScraperComponentConfig{
+				Runtime:        conf.Runtime,
+				Workdir:        env.workdir,
+				Image:          conf.DashboardMetricsScraperImage,
+				KubeconfigPath: env.inClusterOnHostKubeconfigPath,
+				CaCertPath:     env.caCertPath,
+				AdminCertPath:  env.adminCertPath,
+				AdminKeyPath:   env.adminKeyPath,
+			})
+			if err != nil {
+				return err
+			}
+			dashboardMetricsScraperPod, err := yaml.Marshal(components.ConvertToPod(dashboardMetricsScraperComponent))
+			if err != nil {
+				return fmt.Errorf("failed to marshal dashboard metrics scraper pod: %w", err)
+			}
+			err = c.WriteFile(path.Join(c.GetWorkdirPath(runtime.ManifestsName), consts.ComponentDashboardMetricsScraper+".yaml"), dashboardMetricsScraperPod)
+			if err != nil {
+				return fmt.Errorf("failed to write: %w", err)
+			}
+			env.kwokctlConfig.Components = append(env.kwokctlConfig.Components, dashboardMetricsScraperComponent)
+		}
 	}
 	return nil
 }

pkg/kwokctl/runtime/kind/kind.yaml.tpl

@@ -11,7 +11,7 @@ nodes:
   {{ if or .DashboardPort .PrometheusPort .KwokControllerPort .EtcdPort .JaegerPort}}
   extraPortMappings:
   {{ if .DashboardPort }}
-  - containerPort: 8000
+  - containerPort: 8080
     hostPort: {{ .DashboardPort }}
     protocol: TCP
   {{ end }}

test/kwokctl/kwokctl_workable_test.sh

@@ -206,9 +206,9 @@ function main() {
     minor="${minor%.*}"
 
     if [[ $minor -lt 22 ]]; then
-      create_cluster "${name}" "${release}" -v=debug --enable-metrics-server --prometheus-port 9090 --controller-port 10247 --etcd-port=2400 --kube-scheduler-port=10250 --kube-controller-manager-port=10260 --dashboard-port 8000
+      create_cluster "${name}" "${release}" -v=debug --enable-metrics-server --prometheus-port 9090 --controller-port 10247 --etcd-port=2400 --kube-scheduler-port=10250 --kube-controller-manager-port=10260 --dashboard-port=8080
     else
-      create_cluster "${name}" "${release}" -v=debug --enable-metrics-server --prometheus-port 9090 --jaeger-port 16686 --controller-port 10247 --etcd-port=2400 --kube-scheduler-port=10250 --kube-controller-manager-port=10260 --dashboard-port 8000
+      create_cluster "${name}" "${release}" -v=debug --enable-metrics-server --prometheus-port 9090 --jaeger-port 16686 --controller-port 10247 --etcd-port=2400 --kube-scheduler-port=10250 --kube-controller-manager-port=10260 --dashboard-port=8080
     fi
 
     test_workable "${name}" || failed+=("workable_${name}")