🌱 improve gomodcheck

Signed-off-by: Alexandre Mahdhaoui <alexandre.mahdhaoui@gmail.com>

Makefile

@@ -92,9 +92,9 @@ GOLANGCI_LINT_PKG := github.com/golangci/golangci-lint/cmd/golangci-lint
 $(GOLANGCI_LINT): # Build golangci-lint from tools folder.
 	GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(GOLANGCI_LINT_PKG) $(GOLANGCI_LINT_BIN) $(GOLANGCI_LINT_VER)
 
-GO_MOD_CHECK_DIR := $(abspath ./tools/cmd/gomodcheck)
+GO_MOD_CHECK_DIR := $(abspath ./hack/tools/cmd/gomodcheck)
 GO_MOD_CHECK := $(abspath $(TOOLS_BIN_DIR)/gomodcheck)
-GO_MOD_CHECK_IGNORE := $(abspath ./.gomodcheck.ignore)
+GO_MOD_CHECK_IGNORE := $(abspath ./hack/.gomodcheck.ignore)
 $(GO_MOD_CHECK): # Build gomodcheck
 	go build -C $(GO_MOD_CHECK_DIR) -o $(GO_MOD_CHECK)
 

go.sum

@@ -101,10 +101,12 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
 github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
-github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
+github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
+github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
 github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
+github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM=
+github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
 github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
 github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
 github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=

hack/.gomodcheck.yaml

@@ -0,0 +1,14 @@
+upstreamRefs:
+  - k8s.io/api
+  - k8s.io/apiextensions-apiserver
+  - k8s.io/apimachinery
+  - k8s.io/apiserver
+  - k8s.io/client-go
+  - k8s.io/component-base
+  - k8s.io/klog/v2
+  - k8s.io/utils
+
+excludedModules:
+  # --- test dependencies:
+  - github.com/onsi/ginkgo/v2
+  - github.com/onsi/gomega

hack/tools/cmd/gomodcheck/main.go

@@ -3,70 +3,106 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"io"
-	"net/http"
 	"os"
+	"os/exec"
 	"regexp"
 	"strings"
 
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"go.uber.org/zap"
+	"sigs.k8s.io/yaml"
 )
 
 const (
-	kkModUrl = "https://raw.githubusercontent.com/kubernetes/kubernetes/master/go.mod"
-	modFile  = "./go.mod"
+	modFile = "./go.mod"
 )
 
-var (
-	cleanMods     = regexp.MustCompile(`\t| *//.*`)
-	modDelimStart = regexp.MustCompile(`^require.*`)
-	modDelimEnd   = ")"
-)
+type config struct {
+	UpstreamRefs    []string `yaml:"upstreamRefs"`
+	ExcludedModules []string `yaml:"excludedModules"`
+}
+
+type upstream struct {
+	Ref     string `json:"ref"`
+	Version string `json:"version"`
+}
 
+// representation of an out of sync module
 type oosMod struct {
-	Name            string `json:"name"`
-	Version         string `json:"version"`
-	UpstreamVersion string `json:"upstreamVersion"`
+	Name      string     `json:"name"`
+	Version   string     `json:"version"`
+	Upstreams []upstream `json:"upstreams"`
 }
 
 func main() {
-	logger := zap.New()
-	excludedMods := getExcludedMods()
+	l, _ := zap.NewProduction()
+	logger := l.Sugar()
 
-	// 1. kk
-	resp, err := http.Get(kkModUrl)
-	if err != nil {
-		panic(err.Error())
+	if len(os.Args) < 2 {
+		fmt.Printf("USAGE: %s [PATH_TO_CONFIG_FILE]\n", os.Args[0])
+		os.Exit(1)
 	}
 
-	b, err := io.ReadAll(resp.Body)
+	// --- 1. parse config
+	b, err := os.ReadFile(os.Args[1])
 	if err != nil {
-		panic(err.Error())
+		logger.Fatal(err.Error())
+	}
+
+	cfg := new(config)
+	if err := yaml.Unmarshal(b, cfg); err != nil {
+		logger.Fatal(err.Error())
+	}
+
+	excludedMods := make(map[string]any)
+	for _, mod := range cfg.ExcludedModules {
+		excludedMods[mod] = nil
 	}
 
-	kkMods := parseMod(b)
+	// --- 2. project mods
+	deps, err := parseModFile()
+	if err != nil {
+		logger.Fatal(err.Error())
+	}
 
-	// 2. go.mod
-	b, err = os.ReadFile(modFile)
+	// --- 3. upstream mods (holding upstream refs)
+	upstreamModGraph, err := getUpstreamModGraph(cfg.UpstreamRefs)
 	if err != nil {
-		return
+		logger.Fatal(err.Error())
 	}
 
 	oosMods := make([]oosMod, 0)
 
-	mods := parseMod(b)
-	for k, v := range mods {
-		if _, ok := excludedMods[k]; ok {
-			logger.Info(fmt.Sprintf("skipped module: %s", k))
+	// --- 4. validate
+	// for each module in our project,
+	// if it matches an upstream module,
+	// then for each upstream module,
+	// if project module version doesn't match upstream version,
+	// then we add the version and the ref to the list of out of sync modules.
+	for mod, version := range deps {
+		if _, ok := excludedMods[mod]; ok {
+			logger.Infof("skipped excluded module: %s", mod)
 			continue
 		}
 
-		if upstreamVersion, ok := kkMods[k]; ok && v != upstreamVersion {
-			oosMods = append(oosMods, oosMod{
-				Name:            k,
-				Version:         v,
-				UpstreamVersion: upstreamVersion,
-			})
+		if versionToRef, ok := upstreamModGraph[mod]; ok {
+			upstreams := make([]upstream, 0)
+
+			for upstreamVersion, upstreamRef := range versionToRef {
+				if version != upstreamVersion {
+					upstreams = append(upstreams, upstream{
+						Ref:     upstreamRef,
+						Version: upstreamVersion,
+					})
+				}
+			}
+
+			if len(upstreams) > 0 {
+				oosMods = append(oosMods, oosMod{
+					Name:      mod,
+					Version:   version,
+					Upstreams: upstreams,
+				})
+			}
 		}
 	}
 
@@ -84,7 +120,18 @@ func main() {
 	os.Exit(1)
 }
 
-func parseMod(b []byte) map[string]string {
+var (
+	cleanMods     = regexp.MustCompile(`\t| *//.*`)
+	modDelimStart = regexp.MustCompile(`^require.*`)
+	modDelimEnd   = ")"
+)
+
+func parseModFile() (map[string]string, error) {
+	b, err := os.ReadFile(modFile)
+	if err != nil {
+		return nil, err
+	}
+
 	in := string(cleanMods.ReplaceAll(b, []byte("")))
 	out := make(map[string]string)
 
@@ -94,26 +141,63 @@ func parseMod(b []byte) map[string]string {
 		case modDelimStart.MatchString(s) && !start:
 			start = true
 		case s == modDelimEnd:
-			return out
+			return out, nil
 		case start:
 			kv := strings.SplitN(s, " ", 2)
 			if len(kv) < 2 {
-				panic(fmt.Sprintf("unexpected format for module: %q", s))
+				return nil, fmt.Errorf("unexpected format for module: %q", s)
 			}
 
 			out[kv[0]] = kv[1]
 		}
 	}
 
-	return out
+	return out, nil
 }
 
-func getExcludedMods() map[string]any {
-	out := make(map[string]any)
+func getUpstreamModGraph(upstreamRefs []string) (map[string]map[string]string, error) {
+	b, err := exec.Command("go", "mod", "graph").Output()
+	if err != nil {
+		return nil, err
+	}
+
+	graph := string(b)
+	o1Refs := make(map[string]bool)
+	for _, upstreamRef := range upstreamRefs {
+		o1Refs[upstreamRef] = false
+	}
+
+	modToVersionToUpstreamRef := make(map[string]map[string]string)
+
+	for _, line := range strings.Split(graph, "\n") {
+		upstreamRef := strings.SplitN(line, "@", 2)[0]
+		if _, ok := o1Refs[upstreamRef]; ok {
+			o1Refs[upstreamRef] = true
+			kv := strings.SplitN(strings.SplitN(line, " ", 2)[1], "@", 2)
+			name := kv[0]
+			version := kv[1]
+
+			if m, ok := modToVersionToUpstreamRef[kv[0]]; ok {
+				m[version] = upstreamRef
+			} else {
+				versionToRef := map[string]string{version: upstreamRef}
+				modToVersionToUpstreamRef[name] = versionToRef
+			}
+		}
+	}
+
+	notFound := ""
+	for ref, found := range o1Refs {
+		if !found {
+			notFound = fmt.Sprintf("%s%s, ", notFound, ref)
+		}
+	}
 
-	for _, mod := range os.Args[1:] {
-		out[mod] = nil
+	if notFound != "" {
+		return nil, fmt.Errorf("cannot verify modules;"+
+			"the following specified upstream module cannot be found in go.mod: [ %s ]",
+			strings.TrimSuffix(notFound, ", "))
 	}
 
-	return out
+	return modToVersionToUpstreamRef, nil
 }