diff --git a/pkg/kwokctl/runtime/binary/cluster.go b/pkg/kwokctl/runtime/binary/cluster.go index 5bc4b84a53..069c9fa685 100644 --- a/pkg/kwokctl/runtime/binary/cluster.go +++ b/pkg/kwokctl/runtime/binary/cluster.go @@ -435,6 +435,7 @@ func (c *Cluster) Install(ctx context.Context) error { KubeControllerManagerPort: conf.KubeControllerManagerPort, KubeSchedulerPort: conf.KubeSchedulerPort, KwokControllerPort: conf.KwokControllerPort, + Metrics: runtime.GetMetrics(ctx), }) if err != nil { return fmt.Errorf("failed to generate prometheus yaml: %w", err) diff --git a/pkg/kwokctl/runtime/binary/prometheus.go b/pkg/kwokctl/runtime/binary/prometheus.go index 68bbf769ac..d9443bb48f 100644 --- a/pkg/kwokctl/runtime/binary/prometheus.go +++ b/pkg/kwokctl/runtime/binary/prometheus.go @@ -21,6 +21,8 @@ import ( "fmt" "text/template" + "sigs.k8s.io/kwok/pkg/apis/internalversion" + _ "embed" ) @@ -51,4 +53,5 @@ type BuildPrometheusConfig struct { KubeControllerManagerPort uint32 KubeSchedulerPort uint32 KwokControllerPort uint32 + Metrics []*internalversion.Metric } diff --git a/pkg/kwokctl/runtime/binary/prometheus.yaml.tpl b/pkg/kwokctl/runtime/binary/prometheus.yaml.tpl index a77680f476..def88b325b 100644 --- a/pkg/kwokctl/runtime/binary/prometheus.yaml.tpl +++ b/pkg/kwokctl/runtime/binary/prometheus.yaml.tpl @@ -39,7 +39,18 @@ scrape_configs: static_configs: - targets: - localhost:{{ .KwokControllerPort }} - +{{ $kwokControllerPort := .KwokControllerPort }} +{{ range .Metrics }} +- job_name: "kwok-metric-{{ .Name }}" + scheme: http + honor_timestamps: true + metrics_path: {{ .Spec.Path }} + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - localhost:{{ $kwokControllerPort }} +{{ end }} {{ if .SecurePort }} - job_name: "kube-apiserver" scheme: https diff --git a/pkg/kwokctl/runtime/compose/cluster.go b/pkg/kwokctl/runtime/compose/cluster.go index d19ae570ac..d8205b8a75 100644 --- a/pkg/kwokctl/runtime/compose/cluster.go +++ b/pkg/kwokctl/runtime/compose/cluster.go @@ -403,6 +403,7 @@ func (c *Cluster) Install(ctx context.Context) error { SecurePort: conf.SecurePort, AdminCrtPath: inClusterAdminCertPath, AdminKeyPath: inClusterAdminKeyPath, + Metrics: runtime.GetMetrics(ctx), }) if err != nil { return fmt.Errorf("failed to generate prometheus yaml: %w", err) @@ -483,7 +484,7 @@ func (c *Cluster) Install(ctx context.Context) error { return err } - err = os.WriteFile(inClusterOnHostKubeconfigPath, inClusterKubeconfigData, 0640) + err = os.WriteFile(inClusterOnHostKubeconfigPath, []byte(inClusterKubeconfigData), 0640) if err != nil { return err } diff --git a/pkg/kwokctl/runtime/compose/prometheus.go b/pkg/kwokctl/runtime/compose/prometheus.go index 74aa78af7a..54fcebb57a 100644 --- a/pkg/kwokctl/runtime/compose/prometheus.go +++ b/pkg/kwokctl/runtime/compose/prometheus.go @@ -21,6 +21,8 @@ import ( "fmt" "text/template" + "sigs.k8s.io/kwok/pkg/apis/internalversion" + _ "embed" ) @@ -45,4 +47,5 @@ type BuildPrometheusConfig struct { SecurePort bool AdminCrtPath string AdminKeyPath string + Metrics []*internalversion.Metric } diff --git a/pkg/kwokctl/runtime/compose/prometheus.yaml.tpl b/pkg/kwokctl/runtime/compose/prometheus.yaml.tpl index a35b662784..36717024e7 100644 --- a/pkg/kwokctl/runtime/compose/prometheus.yaml.tpl +++ b/pkg/kwokctl/runtime/compose/prometheus.yaml.tpl @@ -40,6 +40,18 @@ scrape_configs: - targets: - "{{ .ProjectName }}-kwok-controller:10247" +{{ $projectName := .ProjectName }} +{{ range .Metrics }} +- job_name: "kwok-metric-{{ .Name }}" + scheme: http + honor_timestamps: true + metrics_path: {{ .Spec.Path }} + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - "{{ $projectName }}-kwok-controller:10247" +{{ end }} {{ if .SecurePort }} - job_name: "kube-apiserver" scheme: https diff --git a/pkg/kwokctl/runtime/kind/cluster.go b/pkg/kwokctl/runtime/kind/cluster.go index 0a812e532d..793132ead8 100644 --- a/pkg/kwokctl/runtime/kind/cluster.go +++ b/pkg/kwokctl/runtime/kind/cluster.go @@ -194,6 +194,7 @@ func (c *Cluster) Install(ctx context.Context) error { Name: c.Name(), ExtraArgs: prometheusPatches.ExtraArgs, ExtraVolumes: prometheusPatches.ExtraVolumes, + Metrics: runtime.GetMetrics(ctx), } if verbosity != log.LevelInfo { prometheusConf.LogLevel = log.ToLogSeverityLevel(verbosity) diff --git a/pkg/kwokctl/runtime/kind/prometheus_deployment.go b/pkg/kwokctl/runtime/kind/prometheus_deployment.go index d08c4c2e82..22eba106a5 100644 --- a/pkg/kwokctl/runtime/kind/prometheus_deployment.go +++ b/pkg/kwokctl/runtime/kind/prometheus_deployment.go @@ -56,4 +56,5 @@ type BuildPrometheusDeploymentConfig struct { LogLevel string ExtraArgs []internalversion.ExtraArgs ExtraVolumes []internalversion.Volume + Metrics []*internalversion.Metric } diff --git a/pkg/kwokctl/runtime/kind/prometheus_deployment.yaml.tpl b/pkg/kwokctl/runtime/kind/prometheus_deployment.yaml.tpl index 5e32867d35..b70454811b 100644 --- a/pkg/kwokctl/runtime/kind/prometheus_deployment.yaml.tpl +++ b/pkg/kwokctl/runtime/kind/prometheus_deployment.yaml.tpl @@ -77,6 +77,17 @@ data: static_configs: - targets: - "localhost:10247" + {{ range .Metrics }} + - job_name: "kwok-metric-{{ .Name }}" + scheme: http + honor_timestamps: true + metrics_path: {{ .Spec.Path }} + follow_redirects: true + enable_http2: true + static_configs: + - targets: + - "localhost:10247" + {{ end }} - job_name: "kube-apiserver" scheme: https honor_timestamps: true diff --git a/pkg/kwokctl/runtime/util.go b/pkg/kwokctl/runtime/util.go index 203c75182e..771b65fccd 100644 --- a/pkg/kwokctl/runtime/util.go +++ b/pkg/kwokctl/runtime/util.go @@ -98,3 +98,8 @@ func GetLogVolumes(ctx context.Context) []internalversion.Volume { } return volumes } + +// GetMetrics returns metrics for Metrics resource. +func GetMetrics(ctx context.Context) []*internalversion.Metric { + return config.FilterWithTypeFromContext[*internalversion.Metric](ctx) +} diff --git a/test/kwokctl/kwokctl_metric_test.sh b/test/kwokctl/kwokctl_metric_test.sh index ef8a835384..28636930e9 100755 --- a/test/kwokctl/kwokctl_metric_test.sh +++ b/test/kwokctl/kwokctl_metric_test.sh @@ -51,7 +51,7 @@ function test_apply_node_and_pod() { echo "Error: fake-deployment apply failed" return 1 fi - if ! kwokctl --name "${name}" kubectl wait pod -A --all --for=condition=Ready --timeout=12s; then + if ! kwokctl --name "${name}" kubectl wait pod -A --all --for=condition=Ready --timeout=60s; then echo "Error: fake-pod wait failed" echo kwokctl --name "${name}" kubectl get pod -A --all kwokctl --name "${name}" kubectl get pod -A --all