Add presubmit to check stable metrics are stable

Add presubmit to check stable metrics are stable

internal/store/node.go

@@ -79,10 +79,11 @@ func createNodeCreatedFamilyGenerator() generator.FamilyGenerator {
 }
 
 func createNodeInfoFamilyGenerator() generator.FamilyGenerator {
-	return *generator.NewFamilyGenerator(
+	return *generator.NewFamilyGeneratorWithStability(
 		"kube_node_info",
 		"Information about a cluster node.",
 		metric.Gauge,
+		generator.STABLE,
 		"",
 		wrapNodeFunc(func(n *v1.Node) *metric.Family {
 			labelKeys := []string{
@@ -410,10 +411,11 @@ func createNodeStatusCapacityFamilyGenerator() generator.FamilyGenerator {
 // customized condition for cluster node (e.g. node-problem-detector), and
 // Kubernetes may add new core conditions in future.
 func createNodeStatusConditionFamilyGenerator() generator.FamilyGenerator {
-	return *generator.NewFamilyGenerator(
+	return *generator.NewFamilyGeneratorWithStability(
 		"kube_node_status_condition",
 		"The condition of a cluster node.",
 		metric.Gauge,
+		generator.STABLE,
 		"",
 		wrapNodeFunc(func(n *v1.Node) *metric.Family {
 			ms := make([]*metric.Metric, len(n.Status.Conditions)*len(conditionStatuses))

internal/store/stable_metrics_test.go

@@ -0,0 +1,108 @@
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package store
+
+import (
+	_ "embed"
+	"flag"
+	"os"
+	"testing"
+
+	v1 "k8s.io/api/core/v1"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator"
+)
+
+var (
+	update = flag.Bool("update", false, "update the golden files")
+)
+
+func filterStableMetrics(familyGenerator []generator.FamilyGenerator) []generator.FamilyGenerator {
+	filtered := []generator.FamilyGenerator{}
+	for _, f := range familyGenerator {
+		if f.StabilityLevel == generator.STABLE {
+			filtered = append(filtered, f)
+		}
+	}
+	return filtered
+}
+
+func TestStableMetrics(t *testing.T) {
+	flag.Parse()
+	cases := []generateStableMetricsTestCase{
+		{
+			Name: "Node stable metrics",
+			FilePath: "../../tests/testdata/stable_node_metrics.txt",
+			generateMetricsTestCase: generateMetricsTestCase{
+				DropHelp: true,
+				Obj: &v1.Node{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "",
+					},
+					Status: v1.NodeStatus{
+						Conditions: []v1.NodeCondition{
+							{Type: v1.NodeNetworkUnavailable},
+						},
+					},
+				},
+				Want: ``, // will be replaced by data from filePath
+			},
+		},
+	}
+
+	for i, cs := range cases {
+		var err error
+		c := cs.generateMetricsTestCase
+
+		metrics := filterStableMetrics(nodeMetricFamilies(nil, nil))
+		c.Func = generator.ComposeMetricGenFuncs(metrics)
+		c.Headers = generator.ExtractMetricFamilyHeaders(metrics)
+		path := cs.FilePath
+
+		if *update {
+			writeFile(c.runWithOutput(), path)
+		} else {
+			c.Want, err = readFile(path)
+			if err != nil {
+				t.Errorf("Can't open file %v", path)
+			}
+			err = c.run()
+			if err != nil {
+				t.Errorf("unexpected collecting result in %vth run:\n%s", i, err)
+			}
+		}
+	}
+}
+
+func readFile(fileName string) (string, error) {
+	b, err := os.ReadFile(fileName)
+	if err != nil {
+		return "", err
+	}
+	return string(b), err
+}
+
+func writeFile(data string, fileName string) {
+	file, err := os.Create(fileName)
+	if err != nil {
+		return
+	}
+	defer file.Close()
+
+	file.WriteString(data)
+}

internal/store/testutils.go

@@ -37,9 +37,32 @@ type generateMetricsTestCase struct {
 	Want                 string
 	Headers              []string
 	Func                 func(interface{}) []metric.FamilyInterface
+	DropHelp             bool
+}
+
+type generateStableMetricsTestCase struct {
+	Name string
+	FilePath string
+	generateMetricsTestCase
 }
 
 func (testCase *generateMetricsTestCase) run() error {
+	out := testCase.generateOut()
+	testCase.Want, out = alignResult(testCase.Want, out)
+
+	if diff := cmp.Diff(testCase.Want, out); diff != "" {
+		return fmt.Errorf("expected wanted output to equal output: %w", fmt.Errorf("(-want, +got):\n%s", diff))
+	}
+	return nil
+}
+
+func (testCase *generateMetricsTestCase) runWithOutput() string {
+	out := testCase.generateOut()
+	out, _ = alignResult(out, out)
+	return out
+}
+
+func (testCase *generateMetricsTestCase) generateOut() string {
 	metricFamilies := testCase.Func(testCase.Obj)
 	metricFamilyStrings := []string{}
 	for _, f := range metricFamilies {
@@ -48,31 +71,24 @@ func (testCase *generateMetricsTestCase) run() error {
 	metric := strings.Split(strings.Join(metricFamilyStrings, ""), "\n")
 	filteredMetrics := filterMetricNames(metric, testCase.MetricNames)
 	filteredHeaders := filterMetricNames(testCase.Headers, testCase.MetricNames)
+	if testCase.DropHelp {
+		filteredHeaders = dropHELP(filteredHeaders)
+	}
 	headers := strings.Join(filteredHeaders, "\n")
 	metrics := strings.Join(filteredMetrics, "\n")
 	out := headers + "\n" + metrics
-
-	if err := compareOutput(testCase.Want, out); err != nil {
-		return fmt.Errorf("expected wanted output to equal output: %w", err)
-	}
-
-	return nil
+	return out
 }
 
-func compareOutput(expected, actual string) error {
+func alignResult(expected, actual string) (string, string) {
 	entities := []string{expected, actual}
 	// Align wanted and actual
 	for i := 0; i < len(entities); i++ {
 		for _, f := range []func(string) string{removeUnusedWhitespace, sortLabels, sortByLine} {
 			entities[i] = f(entities[i])
 		}
 	}
-
-	if diff := cmp.Diff(entities[0], entities[1]); diff != "" {
-		return fmt.Errorf("(-want, +got):\n%s", diff)
-	}
-
-	return nil
+	return entities[0], entities[1]
 }
 
 // sortLabels sorts the order of labels in each line of the given metric. The
@@ -141,6 +157,17 @@ func filterMetricNames(ms []string, names []string) []string {
 	return filtered
 }
 
+func dropHELP(header []string) []string {
+	filtered := []string{}
+	for _, ms := range header {
+		m := strings.Split(ms, "\n")
+		if len(m) == 2 && m[1][:6] == "# TYPE" {
+		  filtered = append(filtered, m[1])
+	  }
+	}
+	return filtered
+}
+
 func removeUnusedWhitespace(s string) string {
 	var (
 		trimmedLine  string

pkg/metric_generator/generator.go

@@ -33,16 +33,33 @@ type FamilyGenerator struct {
 	Type              metric.Type
 	OptIn             bool
 	DeprecatedVersion string
+	StabilityLevel    StabilityLevel
 	GenerateFunc      func(obj interface{}) *metric.Family
 }
 
-// NewFamilyGenerator creates new FamilyGenerator instances.
-func NewFamilyGenerator(name string, help string, metricType metric.Type, deprecatedVersion string, generateFunc func(obj interface{}) *metric.Family) *FamilyGenerator {
+// StabilityLevel represents the API guarantees for a given defined metric.
+type StabilityLevel string
+
+const (
+	// For metrics haven't migrated into NewFamilyGeneratorWithStability. Its
+	// level is UNKNOWN. You can find its metric level from documentation.
+	UNKNOWN StabilityLevel = "UNKNOWN"
+	// ALPHA metrics have no stability guarantees, as such, labels may
+	// be arbitrarily added/removed and the metric may be deleted at any time.
+	EXPERIMENTAL StabilityLevel = "EXPERIMENTAL"
+	// STABLE metrics are guaranteed that metric name and labels are same.
+	STABLE StabilityLevel = "STABLE"
+)
+
+// NewFamilyGenerator creates new FamilyGenerator instances with metric
+// stabilityLevel.
+func NewFamilyGeneratorWithStability(name string, help string, metricType metric.Type, stabilityLevel StabilityLevel, deprecatedVersion string, generateFunc func(obj interface{}) *metric.Family) *FamilyGenerator {
 	f := &FamilyGenerator{
 		Name:              name,
 		Type:              metricType,
 		Help:              help,
 		OptIn:             false,
+		StabilityLevel:    stabilityLevel,
 		DeprecatedVersion: deprecatedVersion,
 		GenerateFunc:      generateFunc,
 	}
@@ -52,6 +69,11 @@ func NewFamilyGenerator(name string, help string, metricType metric.Type, deprec
 	return f
 }
 
+// NewFamilyGenerator creates new FamilyGenerator instances.
+func NewFamilyGenerator(name string, help string, metricType metric.Type, deprecatedVersion string, generateFunc func(obj interface{}) *metric.Family) *FamilyGenerator {
+	return NewFamilyGeneratorWithStability(name, help, metricType, UNKNOWN, deprecatedVersion, generateFunc)
+}
+
 // NewOptInFamilyGenerator creates new FamilyGenerator instances for opt-in metric families.
 func NewOptInFamilyGenerator(name string, help string, metricType metric.Type, deprecatedVersion string, generateFunc func(obj interface{}) *metric.Family) *FamilyGenerator {
 	f := NewFamilyGenerator(name, help, metricType, deprecatedVersion, generateFunc)

tests/testdata/stable_node_metrics.txt

@@ -0,0 +1,6 @@
+# TYPE kube_node_info gauge
+# TYPE kube_node_status_condition gauge
+kube_node_info{container_runtime_version="",internal_ip="",kernel_version="",kubelet_version="",kubeproxy_version="",node="",os_image="",pod_cidr="",provider_id="",system_uuid=""} 1
+kube_node_status_condition{condition="NetworkUnavailable",node="",status="false"} 0
+kube_node_status_condition{condition="NetworkUnavailable",node="",status="true"} 0
+kube_node_status_condition{condition="NetworkUnavailable",node="",status="unknown"} 0