Included paths for prometheus collection targets

kubernetes-sigs · Jun 12, 2023 · 077008e · 077008e
1 parent cbe0554
commit 077008e
Show file tree

Hide file tree

Showing 11 changed files with 52 additions and 3 deletions.
diff --git a/pkg/kwokctl/runtime/binary/cluster.go b/pkg/kwokctl/runtime/binary/cluster.go
@@ -435,6 +435,7 @@ func (c *Cluster) Install(ctx context.Context) error {
 			KubeControllerManagerPort: conf.KubeControllerManagerPort,
 			KubeSchedulerPort:         conf.KubeSchedulerPort,
 			KwokControllerPort:        conf.KwokControllerPort,
+			Metrics:                   runtime.GetMetrics(ctx),
 		})
 		if err != nil {
 			return fmt.Errorf("failed to generate prometheus yaml: %w", err)

diff --git a/pkg/kwokctl/runtime/binary/prometheus.go b/pkg/kwokctl/runtime/binary/prometheus.go
@@ -21,6 +21,8 @@ import (
 	"fmt"
 	"text/template"
 
+	"sigs.k8s.io/kwok/pkg/apis/internalversion"
+
 	_ "embed"
 )
 
@@ -51,4 +53,5 @@ type BuildPrometheusConfig struct {
 	KubeControllerManagerPort uint32
 	KubeSchedulerPort         uint32
 	KwokControllerPort        uint32
+	Metrics                   []*internalversion.Metric
 }
diff --git a/pkg/kwokctl/runtime/binary/prometheus.yaml.tpl b/pkg/kwokctl/runtime/binary/prometheus.yaml.tpl
@@ -39,7 +39,18 @@ scrape_configs:
   static_configs:
   - targets:
     - localhost:{{ .KwokControllerPort }}
-
+{{ $kwokControllerPort := .KwokControllerPort }}
+{{ range .Metrics }}
+- job_name: "kwok-metric-{{ .Name }}"
+  scheme: http
+  honor_timestamps: true
+  metrics_path: {{ .Spec.Path }}
+  follow_redirects: true
+  enable_http2: true
+  static_configs:
+  - targets:
+    - localhost:{{ $kwokControllerPort }}
+{{ end }}
 {{ if .SecurePort }}
 - job_name: "kube-apiserver"
   scheme: https

diff --git a/pkg/kwokctl/runtime/compose/cluster.go b/pkg/kwokctl/runtime/compose/cluster.go
@@ -403,6 +403,7 @@ func (c *Cluster) Install(ctx context.Context) error {
 			SecurePort:   conf.SecurePort,
 			AdminCrtPath: inClusterAdminCertPath,
 			AdminKeyPath: inClusterAdminKeyPath,
+			Metrics:      runtime.GetMetrics(ctx),
 		})
 		if err != nil {
 			return fmt.Errorf("failed to generate prometheus yaml: %w", err)
@@ -483,7 +484,7 @@ func (c *Cluster) Install(ctx context.Context) error {
 		return err
 	}
 
-	err = os.WriteFile(inClusterOnHostKubeconfigPath, inClusterKubeconfigData, 0640)
+	err = os.WriteFile(inClusterOnHostKubeconfigPath, []byte(inClusterKubeconfigData), 0640)
 	if err != nil {
 		return err
 	}

diff --git a/pkg/kwokctl/runtime/compose/prometheus.go b/pkg/kwokctl/runtime/compose/prometheus.go
@@ -21,6 +21,8 @@ import (
 	"fmt"
 	"text/template"
 
+	"sigs.k8s.io/kwok/pkg/apis/internalversion"
+
 	_ "embed"
 )
 
@@ -45,4 +47,5 @@ type BuildPrometheusConfig struct {
 	SecurePort   bool
 	AdminCrtPath string
 	AdminKeyPath string
+	Metrics      []*internalversion.Metric
 }
diff --git a/pkg/kwokctl/runtime/compose/prometheus.yaml.tpl b/pkg/kwokctl/runtime/compose/prometheus.yaml.tpl
@@ -40,6 +40,18 @@ scrape_configs:
   - targets:
     - "{{ .ProjectName }}-kwok-controller:10247"
 
+{{ $projectName := .ProjectName }}
+{{ range .Metrics }}
+- job_name: "kwok-metric-{{ .Name }}"
+  scheme: http
+  honor_timestamps: true
+  metrics_path: {{ .Spec.Path }}
+  follow_redirects: true
+  enable_http2: true
+  static_configs:
+  - targets:
+    - "{{ $projectName }}-kwok-controller:10247"
+{{ end }}
 {{ if .SecurePort }}
 - job_name: "kube-apiserver"
   scheme: https

diff --git a/pkg/kwokctl/runtime/kind/cluster.go b/pkg/kwokctl/runtime/kind/cluster.go
@@ -194,6 +194,7 @@ func (c *Cluster) Install(ctx context.Context) error {
 			Name:            c.Name(),
 			ExtraArgs:       prometheusPatches.ExtraArgs,
 			ExtraVolumes:    prometheusPatches.ExtraVolumes,
+			Metrics:         runtime.GetMetrics(ctx),
 		}
 		if verbosity != log.LevelInfo {
 			prometheusConf.LogLevel = log.ToLogSeverityLevel(verbosity)

diff --git a/pkg/kwokctl/runtime/kind/prometheus_deployment.go b/pkg/kwokctl/runtime/kind/prometheus_deployment.go
@@ -56,4 +56,5 @@ type BuildPrometheusDeploymentConfig struct {
 	LogLevel        string
 	ExtraArgs       []internalversion.ExtraArgs
 	ExtraVolumes    []internalversion.Volume
+	Metrics         []*internalversion.Metric
 }
diff --git a/pkg/kwokctl/runtime/kind/prometheus_deployment.yaml.tpl b/pkg/kwokctl/runtime/kind/prometheus_deployment.yaml.tpl
@@ -77,6 +77,17 @@ data:
         static_configs:
           - targets:
               - "localhost:10247"
+      {{ range .Metrics }}
+      - job_name: "kwok-metric-{{ .Name }}"
+        scheme: http
+        honor_timestamps: true
+        metrics_path: {{ .Spec.Path }}
+        follow_redirects: true
+        enable_http2: true
+        static_configs:
+        - targets:
+            - "localhost:10247"
+      {{ end }}
       - job_name: "kube-apiserver"
         scheme: https
         honor_timestamps: true

diff --git a/pkg/kwokctl/runtime/util.go b/pkg/kwokctl/runtime/util.go
@@ -98,3 +98,8 @@ func GetLogVolumes(ctx context.Context) []internalversion.Volume {
 	}
 	return volumes
 }
+
+// GetMetrics returns metrics for Metrics resource.
+func GetMetrics(ctx context.Context) []*internalversion.Metric {
+	return config.FilterWithTypeFromContext[*internalversion.Metric](ctx)
+}
diff --git a/test/kwokctl/kwokctl_metric_test.sh b/test/kwokctl/kwokctl_metric_test.sh
@@ -51,7 +51,7 @@ function test_apply_node_and_pod() {
     echo "Error: fake-deployment apply failed"
     return 1
   fi
-  if ! kwokctl --name "${name}" kubectl wait pod -A --all --for=condition=Ready --timeout=12s; then
+  if ! kwokctl --name "${name}" kubectl wait pod -A --all --for=condition=Ready --timeout=60s; then
     echo "Error: fake-pod wait failed"
     echo kwokctl --name "${name}" kubectl get pod -A --all
     kwokctl --name "${name}" kubectl get pod -A --all