Included paths for prometheus collection targets

pkg/kwokctl/runtime/binary/cluster.go

@@ -68,41 +68,41 @@ func (c *Cluster) download(ctx context.Context) error {
 	conf := &config.Options
 
 	kubeApiserverPath := c.GetBinPath("kube-apiserver" + conf.BinSuffix)
-	err = file.DownloadWithCache(ctx, conf.CacheDir, conf.KubeApiserverBinary, kubeApiserverPath, 0750, conf.QuietPull)
+	err = file.DownloadWithCache(ctx, conf.CacheDir, conf.KubeApiserverBinary, kubeApiserverPath, 0o750, conf.QuietPull)
 	if err != nil {
 		return err
 	}
 
 	if !conf.DisableKubeControllerManager {
 		kubeControllerManagerPath := c.GetBinPath("kube-controller-manager" + conf.BinSuffix)
-		err = file.DownloadWithCache(ctx, conf.CacheDir, conf.KubeControllerManagerBinary, kubeControllerManagerPath, 0750, conf.QuietPull)
+		err = file.DownloadWithCache(ctx, conf.CacheDir, conf.KubeControllerManagerBinary, kubeControllerManagerPath, 0o750, conf.QuietPull)
 		if err != nil {
 			return err
 		}
 	}
 
 	if !conf.DisableKubeScheduler {
 		kubeSchedulerPath := c.GetBinPath("kube-scheduler" + conf.BinSuffix)
-		err = file.DownloadWithCache(ctx, conf.CacheDir, conf.KubeSchedulerBinary, kubeSchedulerPath, 0750, conf.QuietPull)
+		err = file.DownloadWithCache(ctx, conf.CacheDir, conf.KubeSchedulerBinary, kubeSchedulerPath, 0o750, conf.QuietPull)
 		if err != nil {
 			return err
 		}
 	}
 
 	kwokControllerPath := c.GetBinPath("kwok-controller" + conf.BinSuffix)
-	err = file.DownloadWithCache(ctx, conf.CacheDir, conf.KwokControllerBinary, kwokControllerPath, 0750, conf.QuietPull)
+	err = file.DownloadWithCache(ctx, conf.CacheDir, conf.KwokControllerBinary, kwokControllerPath, 0o750, conf.QuietPull)
 	if err != nil {
 		return err
 	}
 
 	etcdPath := c.GetBinPath("etcd" + conf.BinSuffix)
 	if conf.EtcdBinary == "" {
-		err = file.DownloadWithCacheAndExtract(ctx, conf.CacheDir, conf.EtcdBinaryTar, etcdPath, "etcd"+conf.BinSuffix, 0750, conf.QuietPull, true)
+		err = file.DownloadWithCacheAndExtract(ctx, conf.CacheDir, conf.EtcdBinaryTar, etcdPath, "etcd"+conf.BinSuffix, 0o750, conf.QuietPull, true)
 		if err != nil {
 			return err
 		}
 	} else {
-		err = file.DownloadWithCache(ctx, conf.CacheDir, conf.EtcdBinary, etcdPath, 0750, conf.QuietPull)
+		err = file.DownloadWithCache(ctx, conf.CacheDir, conf.EtcdBinary, etcdPath, 0o750, conf.QuietPull)
 		if err != nil {
 			return err
 		}
@@ -111,12 +111,12 @@ func (c *Cluster) download(ctx context.Context) error {
 	if conf.PrometheusPort != 0 {
 		prometheusPath := c.GetBinPath("prometheus" + conf.BinSuffix)
 		if conf.PrometheusBinary == "" {
-			err = file.DownloadWithCacheAndExtract(ctx, conf.CacheDir, conf.PrometheusBinaryTar, prometheusPath, "prometheus"+conf.BinSuffix, 0750, conf.QuietPull, true)
+			err = file.DownloadWithCacheAndExtract(ctx, conf.CacheDir, conf.PrometheusBinaryTar, prometheusPath, "prometheus"+conf.BinSuffix, 0o750, conf.QuietPull, true)
 			if err != nil {
 				return err
 			}
 		} else {
-			err = file.DownloadWithCache(ctx, conf.CacheDir, conf.PrometheusBinary, prometheusPath, 0750, conf.QuietPull)
+			err = file.DownloadWithCache(ctx, conf.CacheDir, conf.PrometheusBinary, prometheusPath, 0o750, conf.QuietPull)
 			if err != nil {
 				return err
 			}
@@ -143,7 +143,7 @@ func (c *Cluster) setup(ctx context.Context) error {
 
 	if conf.KubeAuditPolicy != "" {
 		auditLogPath := c.GetLogPath(runtime.AuditLogName)
-		err = file.Create(auditLogPath, 0644)
+		err = file.Create(auditLogPath, 0o644)
 		if err != nil {
 			return err
 		}
@@ -156,7 +156,7 @@ func (c *Cluster) setup(ctx context.Context) error {
 	}
 
 	etcdDataPath := c.GetWorkdirPath(runtime.EtcdDataDirName)
-	err = os.MkdirAll(etcdDataPath, 0750)
+	err = os.MkdirAll(etcdDataPath, 0o750)
 	if err != nil {
 		return fmt.Errorf("failed to mkdir etcd data path: %w", err)
 	}
@@ -420,12 +420,13 @@ func (c *Cluster) Install(ctx context.Context) error {
 			KubeControllerManagerPort: conf.KubeControllerManagerPort,
 			KubeSchedulerPort:         conf.KubeSchedulerPort,
 			KwokControllerPort:        conf.KwokControllerPort,
+			Metrics:                   runtime.GetMetrics(ctx),
 		})
 		if err != nil {
 			return fmt.Errorf("failed to generate prometheus yaml: %w", err)
 		}
 		prometheusConfigPath := c.GetWorkdirPath(runtime.Prometheus)
-		err = os.WriteFile(prometheusConfigPath, []byte(prometheusData), 0640)
+		err = os.WriteFile(prometheusConfigPath, []byte(prometheusData), 0o640)
 		if err != nil {
 			return fmt.Errorf("failed to write prometheus yaml: %w", err)
 		}
@@ -464,7 +465,7 @@ func (c *Cluster) Install(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	err = os.WriteFile(kubeconfigPath, []byte(kubeconfigData), 0640)
+	err = os.WriteFile(kubeconfigPath, []byte(kubeconfigData), 0o640)
 	if err != nil {
 		return err
 	}
@@ -690,7 +691,7 @@ func (c *Cluster) Logs(ctx context.Context, name string, out io.Writer) error {
 
 	logs := c.GetLogPath(filepath.Base(name) + ".log")
 
-	f, err := os.OpenFile(logs, os.O_RDONLY, 0640)
+	f, err := os.OpenFile(logs, os.O_RDONLY, 0o640)
 	if err != nil {
 		return err
 	}

pkg/kwokctl/runtime/binary/prometheus.go

@@ -21,6 +21,8 @@ import (
 	"fmt"
 	"text/template"
 
+	"sigs.k8s.io/kwok/pkg/apis/internalversion"
+
 	_ "embed"
 )
 
@@ -51,4 +53,5 @@ type BuildPrometheusConfig struct {
 	KubeControllerManagerPort uint32
 	KubeSchedulerPort         uint32
 	KwokControllerPort        uint32
+	Metrics                   []*internalversion.Metric
 }

pkg/kwokctl/runtime/binary/prometheus.yaml.tpl

@@ -39,7 +39,18 @@ scrape_configs:
   static_configs:
   - targets:
     - localhost:{{ .KwokControllerPort }}
-
+{{ $kwokControllerPort := .KwokControllerPort }}
+{{ range .Metrics }}
+- job_name: "kwok-metric-{{ .Name }}"
+  scheme: http
+  honor_timestamps: true
+  metrics_path: {{ .Spec.Path }}
+  follow_redirects: true
+  enable_http2: true
+  static_configs:
+  - targets:
+    - localhost:{{ $kwokControllerPort }}
+{{ end }}
 {{ if .SecurePort }}
 - job_name: "kube-apiserver"
   scheme: https

pkg/kwokctl/runtime/compose/cluster.go

@@ -96,7 +96,7 @@ func (c *Cluster) setup(ctx context.Context) error {
 
 	if conf.KubeAuditPolicy != "" {
 		auditLogPath := c.GetLogPath(runtime.AuditLogName)
-		err = file.Create(auditLogPath, 0640)
+		err = file.Create(auditLogPath, 0o640)
 		if err != nil {
 			return err
 		}
@@ -109,7 +109,7 @@ func (c *Cluster) setup(ctx context.Context) error {
 	}
 
 	etcdDataPath := c.GetWorkdirPath(runtime.EtcdDataDirName)
-	err = os.MkdirAll(etcdDataPath, 0750)
+	err = os.MkdirAll(etcdDataPath, 0o750)
 	if err != nil {
 		return fmt.Errorf("failed to mkdir etcd data path: %w", err)
 	}
@@ -390,6 +390,7 @@ func (c *Cluster) Install(ctx context.Context) error {
 			SecurePort:   conf.SecurePort,
 			AdminCrtPath: inClusterAdminCertPath,
 			AdminKeyPath: inClusterAdminKeyPath,
+			Metrics:      runtime.GetMetrics(ctx),
 		})
 		if err != nil {
 			return fmt.Errorf("failed to generate prometheus yaml: %w", err)
@@ -398,7 +399,7 @@ func (c *Cluster) Install(ctx context.Context) error {
 		//nolint:gosec
 		// We don't need to check the permissions of the prometheus config file,
 		// because it's working in a non-root container.
-		err = os.WriteFile(prometheusConfigPath, []byte(prometheusData), 0644)
+		err = os.WriteFile(prometheusConfigPath, []byte(prometheusData), 0o644)
 		if err != nil {
 			return fmt.Errorf("failed to write prometheus yaml: %w", err)
 		}
@@ -463,17 +464,17 @@ func (c *Cluster) Install(ctx context.Context) error {
 	}
 
 	// Save config
-	err = os.WriteFile(kubeconfigPath, []byte(kubeconfigData), 0640)
+	err = os.WriteFile(kubeconfigPath, []byte(kubeconfigData), 0o640)
 	if err != nil {
 		return err
 	}
 
-	err = os.WriteFile(inClusterOnHostKubeconfigPath, []byte(inClusterKubeconfigData), 0640)
+	err = os.WriteFile(inClusterOnHostKubeconfigPath, []byte(inClusterKubeconfigData), 0o640)
 	if err != nil {
 		return err
 	}
 
-	err = os.WriteFile(composePath, composeData, 0640)
+	err = os.WriteFile(composePath, composeData, 0o640)
 	if err != nil {
 		return err
 	}
@@ -760,7 +761,7 @@ func (c *Cluster) buildComposeCommands(ctx context.Context, args ...string) ([]s
 		if err != nil {
 			// docker compose subcommand does not exist, try to download it
 			dockerComposePath := c.GetBinPath("docker-compose" + conf.BinSuffix)
-			err = file.DownloadWithCache(ctx, conf.CacheDir, conf.DockerComposeBinary, dockerComposePath, 0750, conf.QuietPull)
+			err = file.DownloadWithCache(ctx, conf.CacheDir, conf.DockerComposeBinary, dockerComposePath, 0o750, conf.QuietPull)
 			if err != nil {
 				return nil, err
 			}

pkg/kwokctl/runtime/compose/prometheus.go

@@ -21,6 +21,8 @@ import (
 	"fmt"
 	"text/template"
 
+	"sigs.k8s.io/kwok/pkg/apis/internalversion"
+
 	_ "embed"
 )
 
@@ -45,4 +47,5 @@ type BuildPrometheusConfig struct {
 	SecurePort   bool
 	AdminCrtPath string
 	AdminKeyPath string
+	Metrics      []*internalversion.Metric
 }

pkg/kwokctl/runtime/compose/prometheus.yaml.tpl

@@ -40,6 +40,18 @@ scrape_configs:
   - targets:
     - "{{ .ProjectName }}-kwok-controller:10247"
 
+{{ $projectName := .ProjectName }}
+{{ range .Metrics }}
+- job_name: "kwok-metric-{{ .Name }}"
+  scheme: http
+  honor_timestamps: true
+  metrics_path: {{ .Spec.Path }}
+  follow_redirects: true
+  enable_http2: true
+  static_configs:
+  - targets:
+    - "{{ $projectName }}-kwok-controller:10247"
+{{ end }}
 {{ if .SecurePort }}
 - job_name: "kube-apiserver"
   scheme: https

pkg/kwokctl/runtime/kind/cluster.go

@@ -109,7 +109,7 @@ func (c *Cluster) Install(ctx context.Context) error {
 	auditPolicyPath := ""
 	if conf.KubeAuditPolicy != "" {
 		auditLogPath = c.GetLogPath(runtime.AuditLogName)
-		err = file.Create(auditLogPath, 0640)
+		err = file.Create(auditLogPath, 0o640)
 		if err != nil {
 			return err
 		}
@@ -166,7 +166,7 @@ func (c *Cluster) Install(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	err = os.WriteFile(c.GetWorkdirPath(runtime.KindName), []byte(kindYaml), 0640)
+	err = os.WriteFile(c.GetWorkdirPath(runtime.KindName), []byte(kindYaml), 0o640)
 	if err != nil {
 		return fmt.Errorf("failed to write %s: %w", runtime.KindName, err)
 	}
@@ -182,7 +182,7 @@ func (c *Cluster) Install(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	err = os.WriteFile(c.GetWorkdirPath(runtime.KwokPod), []byte(kwokControllerPod), 0640)
+	err = os.WriteFile(c.GetWorkdirPath(runtime.KwokPod), []byte(kwokControllerPod), 0o640)
 	if err != nil {
 		return fmt.Errorf("failed to write %s: %w", runtime.KwokPod, err)
 	}
@@ -194,6 +194,7 @@ func (c *Cluster) Install(ctx context.Context) error {
 			Name:            c.Name(),
 			ExtraArgs:       prometheusPatches.ExtraArgs,
 			ExtraVolumes:    prometheusPatches.ExtraVolumes,
+			Metrics:         runtime.GetMetrics(ctx),
 		}
 		if verbosity != log.LevelInfo {
 			prometheusConf.LogLevel = log.ToLogSeverityLevel(verbosity)
@@ -202,7 +203,7 @@ func (c *Cluster) Install(ctx context.Context) error {
 		if err != nil {
 			return err
 		}
-		err = os.WriteFile(c.GetWorkdirPath(runtime.PrometheusDeploy), []byte(prometheusDeploy), 0640)
+		err = os.WriteFile(c.GetWorkdirPath(runtime.PrometheusDeploy), []byte(prometheusDeploy), 0o640)
 		if err != nil {
 			return fmt.Errorf("failed to write %s: %w", runtime.PrometheusDeploy, err)
 		}
@@ -338,7 +339,7 @@ func (c *Cluster) Up(ctx context.Context) error {
 		return err
 	}
 
-	err = os.WriteFile(kubeconfigPath, kubeconfigBuf.Bytes(), 0640)
+	err = os.WriteFile(kubeconfigPath, kubeconfigBuf.Bytes(), 0o640)
 	if err != nil {
 		return err
 	}
@@ -401,7 +402,7 @@ func loadArchiveImages(ctx context.Context, command string, kindCluster string,
 	logger := log.FromContext(ctx)
 	for _, image := range images {
 		archive := path.Join(tmpDir, "image-archive", strings.ReplaceAll(image, ":", "/")+".tar")
-		err := os.MkdirAll(filepath.Dir(archive), 0750)
+		err := os.MkdirAll(filepath.Dir(archive), 0o750)
 		if err != nil {
 			return err
 		}
@@ -708,7 +709,7 @@ func (c *Cluster) preDownloadKind(ctx context.Context) (string, error) {
 	if err != nil {
 		// kind does not exist, try to download it
 		kindPath := c.GetBinPath("kind" + conf.BinSuffix)
-		err = file.DownloadWithCache(ctx, conf.CacheDir, conf.KindBinary, kindPath, 0750, conf.QuietPull)
+		err = file.DownloadWithCache(ctx, conf.CacheDir, conf.KindBinary, kindPath, 0o750, conf.QuietPull)
 		if err != nil {
 			return "", err
 		}

pkg/kwokctl/runtime/kind/prometheus_deployment.go

@@ -56,4 +56,5 @@ type BuildPrometheusDeploymentConfig struct {
 	LogLevel        string
 	ExtraArgs       []internalversion.ExtraArgs
 	ExtraVolumes    []internalversion.Volume
+	Metrics         []*internalversion.Metric
 }

pkg/kwokctl/runtime/kind/prometheus_deployment.yaml.tpl

@@ -77,6 +77,17 @@ data:
         static_configs:
           - targets:
               - "localhost:10247"
+      {{ range .Metrics }}
+      - job_name: "kwok-metric-{{ .Name }}"
+        scheme: http
+        honor_timestamps: true
+        metrics_path: {{ .Spec.Path }}
+        follow_redirects: true
+        enable_http2: true
+        static_configs:
+        - targets:
+            - "localhost:10247"
+      {{ end }}
       - job_name: "kube-apiserver"
         scheme: https
         honor_timestamps: true

pkg/kwokctl/runtime/util.go

@@ -98,3 +98,8 @@ func GetLogVolumes(ctx context.Context) []internalversion.Volume {
 	}
 	return volumes
 }
+
+// GetMetrics returns metrics for Metrics resource.
+func GetMetrics(ctx context.Context) []*internalversion.Metric {
+	return config.FilterWithTypeFromContext[*internalversion.Metric](ctx)
+}

test/kwokctl/kwokctl_metric_test.sh

@@ -51,7 +51,7 @@ function test_apply_node_and_pod() {
     echo "Error: fake-deployment apply failed"
     return 1
   fi
-  if ! kwokctl --name "${name}" kubectl wait pod -A --all --for=condition=Ready --timeout=12s; then
+  if ! kwokctl --name "${name}" kubectl wait pod -A --all --for=condition=Ready --timeout=60s; then
     echo "Error: fake-pod wait failed"
     echo kwokctl --name "${name}" kubectl get pod -A --all
     kwokctl --name "${name}" kubectl get pod -A --all