refactoring

tests/e2e.sh

@@ -15,7 +15,6 @@
 # limitations under the License.
 
 set -e
-set -x
 set -o pipefail
 
 case $(uname -m) in
@@ -164,7 +163,6 @@ mkdir -p ${KUBE_STATE_METRICS_LOG_DIR}
 echo "access kube-state-metrics metrics endpoint"
 curl -s "http://localhost:8001/api/v1/namespaces/kube-system/services/kube-state-metrics:http-metrics/proxy/metrics" >${KUBE_STATE_METRICS_LOG_DIR}/metrics
 
-cat ${KUBE_STATE_METRICS_LOG_DIR}/metrics
 # check stable metrics aren't changed
 go test ./tests/stable/check_test.go --collectedMetricsFile "$(realpath ${KUBE_STATE_METRICS_LOG_DIR}/metrics)"  --stableMetricsFile "$(realpath tests/stable/testdata/stablemetrics.yaml)"
 

tests/stable/check_test.go

@@ -14,24 +14,25 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// To skip this test, put metric name into skipStableMetrics.
 package stable
 
 import (
 	"flag"
 	"fmt"
+	"path/filepath"
 	"sort"
 	"testing"
-
-	"log"
 	"os"
 	"strings"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	prommodel "github.com/prometheus/client_model/go"
 	"github.com/prometheus/common/expfmt"
 	"gopkg.in/yaml.v3"
 
-	"github.com/google/go-cmp/cmp/cmpopts"
+	"k8s.io/klog/v2"
 )
 
 var promText string
@@ -42,8 +43,8 @@ var skipStableMetrics = []string{}
 type Metric struct {
 	Name   string `yaml:"name"`
 	Help   string `yaml:"help"`
-	Type   string
-	Labels []string
+	Type   string `yaml:"type"`
+	Labels []string `yaml:"labels"`
 	// Histogram type
 	Buckets []float64 `yaml:"buckets,omitempty"`
 }
@@ -57,7 +58,9 @@ func TestMain(m *testing.M) {
 
 func TestStableMetrics(t *testing.T) {
 	mf, err := parsePromText(promText)
-	fatal(err)
+	if err != nil {
+		t.Fatalf("Can't parse collected prometheus metrics text. err = %v", err)
+	}
 	collectedStableMetrics := extractStableMetrics(mf)
 	printMetric(collectedStableMetrics)
 
@@ -70,29 +73,22 @@ func TestStableMetrics(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Stable metrics changed: err = %v", err)
 	} else {
-		fmt.Println("passed")
-	}
-
-}
-
-func fatal(err error) {
-	if err != nil {
-		log.Fatalln(err)
+		klog.Infoln("Passed")
 	}
 }
 
 func printMetric(metrics []Metric) {
 	yamlData, err := yaml.Marshal(metrics)
 	if err != nil {
-		fmt.Printf("error while Marshaling. %v", err)
+		klog.Errorf("error while Marshaling. %v", err)
 	}
-	fmt.Println("---begin YAML file---")
-	fmt.Println(string(yamlData))
-	fmt.Println("---end YAML file---")
+	klog.Infoln("---begin YAML file---")
+	klog.Infoln(string(yamlData))
+	klog.Infoln("---end YAML file---")
 }
 
 func parsePromText(path string) (map[string]*prommodel.MetricFamily, error) {
-	reader, err := os.Open(path)
+	reader, err := os.Open(filepath.Clean(path))
 	if err != nil {
 		return nil, err
 	}
@@ -116,32 +112,34 @@ func getBuckets(v *prommodel.MetricFamily) []float64 {
 	return buckets
 }
 
+func getLabels(m *prommodel.Metric) []string {
+	labels := []string{}
+	for _, y := range m.Label {
+		labels = append(labels, y.GetName())
+	}
+	return labels
+}
+
 func extractStableMetrics(mf map[string]*prommodel.MetricFamily) []Metric {
 	metrics := []Metric{}
 	for _, v := range mf {
 		// Find stable metrics
 		if !strings.Contains(*(v.Help), "[STABLE]") {
 			continue
 		}
-
-		m := Metric{
+		metrics = append(metrics, Metric{
 			Name:    *(v.Name),
 			Help:    *(v.Help),
 			Type:    (v.Type).String(),
 			Buckets: getBuckets(v),
-		}
-		labels := []string{}
-		for _, y := range v.Metric[0].Label {
-			labels = append(labels, y.GetName())
-		}
-		m.Labels = labels
-		metrics = append(metrics, m)
+			Labels:  getLabels(v.Metric[0]),
+		})
 	}
 	return metrics
 }
 
 func readYaml(filename string) (*[]Metric, error) {
-	buf, err := os.ReadFile(filename)
+	buf, err := os.ReadFile(filepath.Clean(filename))
 	if err != nil {
 		return nil, err
 	}
@@ -153,11 +151,11 @@ func readYaml(filename string) (*[]Metric, error) {
 	return c, err
 }
 
-func convert2Map(metrics []Metric, skipMap map[string]int) map[string]Metric {
+func convertToMap(metrics []Metric, skipMap map[string]int) map[string]Metric {
 	name2Metric := map[string]Metric{}
 	for _, v := range metrics {
 		if _, ok := skipMap[v.Name]; ok {
-			fmt.Printf("skip, metric %s is in skip list\n", v.Name)
+			klog.Infof("skip, metric %s is in skip list\n", v.Name)
 			continue
 		}
 		name2Metric[v.Name] = v
@@ -180,8 +178,8 @@ func compare(collectedStableMetrics []Metric, expectedStableMetrics []Metric, sk
 	for _, v := range skipStableMetrics {
 		skipMap[v] = 1
 	}
-	collected := convert2Map(collectedStableMetrics, skipMap)
-	expected := convert2Map(expectedStableMetrics, skipMap)
+	collected := convertToMap(collectedStableMetrics, skipMap)
+	expected := convertToMap(expectedStableMetrics, skipMap)
 
 	var ok bool
 
@@ -191,9 +189,11 @@ func compare(collectedStableMetrics []Metric, expectedStableMetrics []Metric, sk
 		if expectedMetric, ok = collected[name]; !ok {
 			return fmt.Errorf("not found stable metric %s", name)
 		}
+		// Ingore Labels field due to ordering issue
 		if diff := cmp.Diff(metric, expectedMetric, cmpopts.IgnoreFields(Metric{}, "Help", "Labels")); diff != "" {
 			return fmt.Errorf("stable metric %s mismatch (-want +got):\n%s", name, diff)
 		}
+		// Compare Labels field after sorting
 		if diff := cmp.Diff(metric.Labels, expectedMetric.Labels, cmpopts.SortSlices(func(l1, l2 string) bool { return l1 > l2 })); diff != "" {
 			return fmt.Errorf("stable metric label %s mismatch (-want +got):\n%s", name, diff)
 		}

tests/stable/testdata/README.md

@@ -0,0 +1,23 @@
+These stable metrics aren't covered in stablemetrics.yaml, because they aren't
+exposed in KSM in the test cluster:
+- kube_cronjob_spec_starting_deadline_seconds
+- kube_cronjob_status_last_schedule_time
+- kube_ingress_tls
+- kube_job_complete
+- kube_job_failed
+- kube_job_spec_active_deadline_seconds
+- kube_job_status_completion_time
+- kube_node_spec_taint
+- kube_persistentvolume_claim_ref
+- kube_pod_completion_time
+- kube_pod_init_container_info
+- kube_pod_init_container_status_ready
+- kube_pod_init_container_status_restarts_total
+- kube_pod_init_container_status_running
+- kube_pod_init_container_status_terminated
+- kube_pod_init_container_status_waiting
+- kube_pod_spec_volumes_persistentvolumeclaims_info
+- kube_pod_spec_volumes_persistentvolumeclaims_readonly
+- kube_pod_status_unschedulable
+- kube_service_spec_external_ip
+- kube_service_status_load_balancer_ingress