Evaluate metrics without startedContainersTotal

pkg/apis/internalversion/metric_type.go

@@ -23,50 +23,50 @@ import (
 // Metric provides metrics configuration.
 type Metric struct {
 	//+k8s:conversion-gen=false
-	metav1.TypeMeta
+	metav1.TypeMeta `json:",inline"`
 	// Standard list metadata.
 	// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
-	metav1.ObjectMeta
+	metav1.ObjectMeta `json:"metadata"`
 	// Spec holds spec for metrics.
-	Spec MetricSpec
+	Spec MetricSpec `json:"spec"`
 }
 
 // MetricSpec holds spec for metrics.
 type MetricSpec struct {
 	// Path is a restful service path.
-	Path string
+	Path string `json:"path"`
 	// Metrics is a list of metric configurations.
-	Metrics []MetricConfig
+	Metrics []MetricConfig `json:"metrics"`
 }
 
 // MetricConfig provides metric configuration to a single metric
 type MetricConfig struct {
 	// Name is the fully-qualified name of the metric.
-	Name string
+	Name string `json:"name"`
 	// Help provides information about this metric.
-	Help string
+	Help string `json:"help"`
 	// Kind is kind of metric (ex. counter, gauge, histogram).
-	Kind string
+	Kind string `json:"kind"`
 	// Labels are metric labels.
-	Labels []MetricLabel
+	Labels []MetricLabel `json:"labels"`
 	// Value is a CEL expression.
-	Value string
+	Value string `json:"value"`
 	// Buckets is a list of buckets for a histogram metric.
-	Buckets []MetricBucket
+	Buckets []MetricBucket `json:"buckets"`
 }
 
 // MetricLabel holds label name and the value of the label.
 type MetricLabel struct {
 	// Name is a label name.
-	Name string
+	Name string `json:"name"`
 	// Value is a CEL expression.
-	Value string
+	Value string `json:"value"`
 }
 
 // MetricBucket is a single bucket for a metric.
 type MetricBucket struct {
 	// Le is less-than or equal.
-	Le string
+	Le string `json:"le"`
 	// Value is a CEL expression.
-	Value string
+	Value string `json:"value"`
 }

pkg/config/config.go

@@ -322,12 +322,12 @@ func Load(ctx context.Context, src ...string) ([]InternalObject, error) {
 
 // Save saves the given objects to the given path.
 func Save(ctx context.Context, path string, objs []InternalObject) error {
-	err := os.MkdirAll(filepath.Dir(path), 0o750)
+	err := os.MkdirAll(filepath.Dir(path), 0750)
 	if err != nil {
 		return err
 	}
 
-	file, err := os.OpenFile(filepath.Clean(path), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o640)
+	file, err := os.OpenFile(filepath.Clean(path), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0640)
 	if err != nil {
 		return err
 	}
@@ -414,10 +414,6 @@ func FilterWithTypeFromContext[T metav1.Object](ctx context.Context) (out []T) {
 	if len(objs) == 0 {
 		return nil
 	}
-	fmt.Println("Filter with type from context")
-	for _, obj := range objs {
-		fmt.Printf("[] %T\n", obj)
-	}
 	return FilterWithType[T](objs)
 }
 

pkg/kwok/cmd/root.go

@@ -19,7 +19,6 @@ package cmd
 
 import (
 	"context"
-	"fmt"
 	"os"
 	"time"
 
@@ -62,7 +61,6 @@ func NewCommand(ctx context.Context) *cobra.Command {
 		SilenceErrors: true,
 		Version:       version.DisplayVersion(),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			fmt.Printf("ARGS=%v\ncmd=%+v\n", args, cmd)
 			return runE(cmd.Context(), flags)
 		},
 	}

pkg/kwok/controllers/node_controller.go

@@ -26,7 +26,6 @@ import (
 	"text/template"
 	"time"
 
-	"github.com/prometheus/client_golang/prometheus"
 	"github.com/wzshiming/cron"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -270,6 +269,7 @@ func (c *NodeController) watchResources(ctx context.Context, opt metav1.ListOpti
 					node := event.Object.(*corev1.Node)
 					if c.need(node) {
 						c.putNodeInfo(node)
+						c.applyMetrics(ctx, node)
 						c.preprocessChan <- node
 						if c.onNodeManagedFunc != nil {
 							err = c.onNodeManagedFunc(ctx, node.Name)
@@ -284,6 +284,7 @@ func (c *NodeController) watchResources(ctx context.Context, opt metav1.ListOpti
 					node := event.Object.(*corev1.Node)
 					if c.need(node) {
 						c.putNodeInfo(node)
+						c.applyMetrics(ctx, node)
 						c.preprocessChan <- node
 					}
 				case watch.Deleted:
@@ -672,21 +673,20 @@ func (c *NodeController) applyMetrics(ctx context.Context, node *corev1.Node) {
 		logger.Warn("Failed to instantiate evaluator for a node", "error", err.Error())
 	}
 
-	reg := kwokmetrics.DefaultMetrics()
+	reg := kwokmetrics.CustomMetric(node.Name)
 	for _, metric := range metrics {
-		for _, m := range metric.Spec.Metrics {
-			var labels prometheus.Labels
-			if m.Labels != nil {
-				labels = make(prometheus.Labels, len(m.Labels))
-			}
-			for _, l := range m.Labels {
-				labels[l.Name] = l.Value
-			}
-
+		for i := range metric.Spec.Metrics {
+			m := &metric.Spec.Metrics[i]
 			switch m.Kind {
 			case kwokmetrics.KindGauge:
-				g, ok := reg.Gauge(m.Name)
-				if !ok {
+				g, err := reg.Gauge(m, node)
+				if err != nil {
+					logger.Warn(
+						"Failed to get/create gauge metric",
+						"metricConfig", metric.Name,
+						"metricName", m.Name,
+						"error", err.Error(),
+					)
 					continue
 				}
 				val, err := eval.Evaluate(m.Value)
@@ -695,18 +695,31 @@ func (c *NodeController) applyMetrics(ctx context.Context, node *corev1.Node) {
 				}
 				g.Set(val)
 			case kwokmetrics.KindCounter:
-				c, ok := reg.Counter(m.Name)
-				if !ok {
+				c, err := reg.Counter(m, node)
+				if err != nil {
+					logger.Warn(
+						"Failed to get/create counter metric",
+						"metricConfig", metric.Name,
+						"metricName", m.Name,
+						"error", err.Error(),
+					)
 					continue
 				}
 				val, err := eval.Evaluate(m.Value)
 				if err != nil {
 					logger.Warn("Failed to evaluate expression", "expression", m.Value)
+					continue
 				}
 				c.Add(val)
 			case kwokmetrics.KindHistogram:
-				h, ok := reg.Histogram(m.Name)
-				if !ok {
+				h, err := reg.Histogram(m, node)
+				if err != nil {
+					logger.Warn(
+						"Failed to get/create histogram metric",
+						"metricConfig", metric.Name,
+						"metricName", m.Name,
+						"error", err.Error(),
+					)
 					continue
 				}
 				for _, b := range m.Buckets {

pkg/kwok/metrics/cel/evaluate.go

@@ -26,14 +26,14 @@ import (
 	corev1 "k8s.io/api/core/v1"
 )
 
-// MetricsEvaluator is a common interface for all types
+// Evaluator is a common interface for all types
 // that evaluate metrics
-type MetricsEvaluator interface {
+type Evaluator interface {
 	Evaluate(exp string) (float64, error)
 }
 
 // NewNodeEvaluator returns a MetricEvaluator that is able to evaluate node metrics
-func NewNodeEvaluator(node *corev1.Node) (MetricsEvaluator, error) {
+func NewNodeEvaluator(node *corev1.Node) (Evaluator, error) {
 	b, err := json.Marshal(node)
 	if err != nil {
 		return nil, err