Merge pull request #113171 from Jefftree/aggregated-discovery-generic

Aggregated discovery server changes

Kubernetes-commit: 8058e8eff8ba8541ddd48cd54bbb19a3fce62c09

go.mod

@@ -8,16 +8,17 @@ require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/emicklei/go-restful/v3 v3.9.0
 	github.com/gogo/protobuf v1.3.2
+	github.com/google/gofuzz v1.1.0
 	github.com/spf13/cobra v1.6.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.8.0
 	golang.org/x/net v0.1.1-0.20221027164007-c63010009c80
 	k8s.io/api v0.0.0-20221108053748-98c1aa6b3d0a
 	k8s.io/apimachinery v0.0.0-20221108055230-fd8a60496be5
-	k8s.io/apiserver v0.0.0-20221108213719-5643daa2db6a
+	k8s.io/apiserver v0.0.0-20221109094018-807ba65cc7a8
 	k8s.io/client-go v0.0.0-20221108173010-769443557e04
 	k8s.io/code-generator v0.0.0-20221108000200-7429fbb99432
-	k8s.io/component-base v0.0.0-20221108213136-021afb59bb71
+	k8s.io/component-base v0.0.0-20221109013135-4e8a9589a311
 	k8s.io/klog/v2 v2.80.1
 	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280
 	k8s.io/utils v0.0.0-20221107191617-1a15be271d1d
@@ -46,7 +47,6 @@ require (
 	github.com/google/cel-go v0.12.5 // indirect
 	github.com/google/gnostic v0.5.7-v3refs // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
-	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/google/uuid v1.1.2 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
@@ -110,9 +110,9 @@ require (
 replace (
 	k8s.io/api => k8s.io/api v0.0.0-20221108053748-98c1aa6b3d0a
 	k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20221108055230-fd8a60496be5
-	k8s.io/apiserver => k8s.io/apiserver v0.0.0-20221108213719-5643daa2db6a
+	k8s.io/apiserver => k8s.io/apiserver v0.0.0-20221109094018-807ba65cc7a8
 	k8s.io/client-go => k8s.io/client-go v0.0.0-20221108173010-769443557e04
 	k8s.io/code-generator => k8s.io/code-generator v0.0.0-20221108000200-7429fbb99432
-	k8s.io/component-base => k8s.io/component-base v0.0.0-20221108213136-021afb59bb71
+	k8s.io/component-base => k8s.io/component-base v0.0.0-20221109013135-4e8a9589a311
 	k8s.io/kms => k8s.io/kms v0.0.0-20221028080743-a9ba1c11c0c6
 )

go.sum

@@ -726,14 +726,14 @@ k8s.io/api v0.0.0-20221108053748-98c1aa6b3d0a h1:GaCla9HtNyi63kysI/cyeA4bv6wRkIy
 k8s.io/api v0.0.0-20221108053748-98c1aa6b3d0a/go.mod h1:PSXY9/fSNyKgKHUU+O9scnZiW8m+V1znqk49oI6hAEY=
 k8s.io/apimachinery v0.0.0-20221108055230-fd8a60496be5 h1:iFAMJ1evvrO6X7dS7EKujS6An+bp3u/dD6opu8rn0QA=
 k8s.io/apimachinery v0.0.0-20221108055230-fd8a60496be5/go.mod h1:VXMmlsE7YRJ5vyAyWpkKIfFkEbDNpVs0ObpkuQf1WfM=
-k8s.io/apiserver v0.0.0-20221108213719-5643daa2db6a h1:Pgwr1+mXasRCcSKqXNgXPj497HPJjZ5KT8u4GVvNoY0=
-k8s.io/apiserver v0.0.0-20221108213719-5643daa2db6a/go.mod h1:HEuVcGugsLz1DrS8BkNLT7wjhrBVrZh2wFZfTAuy2NE=
+k8s.io/apiserver v0.0.0-20221109094018-807ba65cc7a8 h1:nb4/V/X+Isa/QCEEoMibzBiAsUunnY9TJl9XdKCSH44=
+k8s.io/apiserver v0.0.0-20221109094018-807ba65cc7a8/go.mod h1:D07/28EX92tshRpWdcPpMb4TwlTZNGMh9DKrw5WQ+1E=
 k8s.io/client-go v0.0.0-20221108173010-769443557e04 h1:ad7JkOkiLiyMKWHRkmbJgjCzySdkXxRxWeNosATW0mo=
 k8s.io/client-go v0.0.0-20221108173010-769443557e04/go.mod h1:O6sEWJ2BPd8Dag831LA1lzC3WnE29nuwUJZZ4H2vlyo=
 k8s.io/code-generator v0.0.0-20221108000200-7429fbb99432 h1:lWxXcC+YG3ZY5sCsk337h9RgzUK0sWoNWf8fs6KZm0I=
 k8s.io/code-generator v0.0.0-20221108000200-7429fbb99432/go.mod h1:EzIGoP4u7NcVT4orMRQyvQcWqFCUVApa+V+28R7t/+M=
-k8s.io/component-base v0.0.0-20221108213136-021afb59bb71 h1:Qr7dcMdpWjUZUEkZcbvGGQbtTlVR9b9VqQci/G0jzfY=
-k8s.io/component-base v0.0.0-20221108213136-021afb59bb71/go.mod h1:5bp64lK0p+wJD2BFMin7Akfxiwvt58T4iDz2Q+6woBE=
+k8s.io/component-base v0.0.0-20221109013135-4e8a9589a311 h1:rUbuNcL4yd5fSmGBMgRYsaodvXqHiu9TwjcmqH2toRY=
+k8s.io/component-base v0.0.0-20221109013135-4e8a9589a311/go.mod h1:5bp64lK0p+wJD2BFMin7Akfxiwvt58T4iDz2Q+6woBE=
 k8s.io/gengo v0.0.0-20220902162205-c0856e24416d h1:U9tB195lKdzwqicbJvyJeOXV7Klv+wNAWENRnXEGi08=
 k8s.io/gengo v0.0.0-20220902162205-c0856e24416d/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=

pkg/apiserver/apiserver.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/endpoints/discovery/aggregated"
 	genericfeatures "k8s.io/apiserver/pkg/features"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/egressselector"
@@ -154,6 +155,11 @@ type APIAggregator struct {
 	// openAPIV3AggregationController downloads and caches OpenAPI v3 specs.
 	openAPIV3AggregationController *openapiv3controller.AggregationController
 
+	// discoveryAggregationController downloads and caches discovery documents
+	// from all aggregated apiservices so they are available from /apis endpoint
+	// when discovery with resources are requested
+	discoveryAggregationController DiscoveryAggregationController
+
 	// egressSelector selects the proper egress dialer to communicate with the custom apiserver
 	// overwrites proxyTransport dialer if not nil
 	egressSelector *egressselector.EgressSelector
@@ -244,7 +250,13 @@ func (c completedConfig) NewWithDelegate(delegationTarget genericapiserver.Deleg
 		lister:         s.lister,
 		discoveryGroup: discoveryGroup(enabledVersions),
 	}
-	s.GenericAPIServer.Handler.NonGoRestfulMux.Handle("/apis", apisHandler)
+
+	if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
+		apisHandlerWithAggregationSupport := aggregated.WrapAggregatedDiscoveryToHandler(apisHandler, s.GenericAPIServer.AggregatedDiscoveryGroupManager)
+		s.GenericAPIServer.Handler.NonGoRestfulMux.Handle("/apis", apisHandlerWithAggregationSupport)
+	} else {
+		s.GenericAPIServer.Handler.NonGoRestfulMux.Handle("/apis", apisHandler)
+	}
 	s.GenericAPIServer.Handler.NonGoRestfulMux.UnlistedHandle("/apis/", apisHandler)
 
 	apiserviceRegistrationController := NewAPIServiceRegistrationController(informerFactory.Apiregistration().V1().APIServices(), s)
@@ -365,8 +377,8 @@ func (c completedConfig) NewWithDelegate(delegationTarget genericapiserver.Deleg
 	return s, nil
 }
 
-// PrepareRun prepares the aggregator to run, by setting up the OpenAPI spec and calling
-// the generic PrepareRun.
+// PrepareRun prepares the aggregator to run, by setting up the OpenAPI spec &
+// aggregated discovery document and calling the generic PrepareRun.
 func (s *APIAggregator) PrepareRun() (preparedAPIAggregator, error) {
 	// add post start hook before generic PrepareRun in order to be before /healthz installation
 	if s.openAPIConfig != nil {
@@ -383,6 +395,20 @@ func (s *APIAggregator) PrepareRun() (preparedAPIAggregator, error) {
 		})
 	}
 
+	if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
+		s.discoveryAggregationController = NewDiscoveryManager(
+			s.GenericAPIServer.AggregatedDiscoveryGroupManager,
+		)
+
+		// Setup discovery endpoint
+		s.GenericAPIServer.AddPostStartHookOrDie("apiservice-discovery-controller", func(context genericapiserver.PostStartHookContext) error {
+			// Run discovery manager's worker to watch for new/removed/updated
+			// APIServices to the discovery document can be updated at runtime
+			go s.discoveryAggregationController.Run(context.StopCh)
+			return nil
+		})
+	}
+
 	prepared := s.GenericAPIServer.PrepareRun()
 
 	// delay OpenAPI setup until the delegate had a chance to setup their OpenAPI handlers
@@ -432,6 +458,12 @@ func (s *APIAggregator) AddAPIService(apiService *v1.APIService) error {
 		if s.openAPIV3AggregationController != nil {
 			s.openAPIV3AggregationController.UpdateAPIService(proxyHandler, apiService)
 		}
+		// Forward calls to discovery manager to update discovery document
+		if s.discoveryAggregationController != nil {
+			handlerCopy := *proxyHandler
+			handlerCopy.setServiceAvailable(true)
+			s.discoveryAggregationController.AddAPIService(apiService, &handlerCopy)
+		}
 		return nil
 	}
 
@@ -457,6 +489,10 @@ func (s *APIAggregator) AddAPIService(apiService *v1.APIService) error {
 	if s.openAPIV3AggregationController != nil {
 		s.openAPIV3AggregationController.AddAPIService(proxyHandler, apiService)
 	}
+	if s.discoveryAggregationController != nil {
+		s.discoveryAggregationController.AddAPIService(apiService, proxyHandler)
+	}
+
 	s.proxyHandlers[apiService.Name] = proxyHandler
 	s.GenericAPIServer.Handler.NonGoRestfulMux.Handle(proxyPath, proxyHandler)
 	s.GenericAPIServer.Handler.NonGoRestfulMux.UnlistedHandlePrefix(proxyPath+"/", proxyHandler)
@@ -489,6 +525,11 @@ func (s *APIAggregator) AddAPIService(apiService *v1.APIService) error {
 // RemoveAPIService removes the APIService from being handled.  It is not thread-safe, so only call it on one thread at a time please.
 // It's a slow moving API, so it's ok to run the controller on a single thread.
 func (s *APIAggregator) RemoveAPIService(apiServiceName string) {
+	// Forward calls to discovery manager to update discovery document
+	if s.discoveryAggregationController != nil {
+		s.discoveryAggregationController.RemoveAPIService(apiServiceName)
+	}
+
 	version := v1helper.APIServiceNameToGroupVersion(apiServiceName)
 
 	proxyPath := "/apis/" + version.Group + "/" + version.Version

pkg/apiserver/handler_apis.go

@@ -91,7 +91,7 @@ func (r *apisHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		}
 	}
 
-	responsewriters.WriteObjectNegotiated(r.codecs, negotiation.DefaultEndpointRestrictions, schema.GroupVersion{}, w, req, http.StatusOK, discoveryGroupList)
+	responsewriters.WriteObjectNegotiated(r.codecs, negotiation.DefaultEndpointRestrictions, schema.GroupVersion{}, w, req, http.StatusOK, discoveryGroupList, false)
 }
 
 // convertToDiscoveryAPIGroup takes apiservices in a single group and returns a discovery compatible object.
@@ -162,5 +162,5 @@ func (r *apiGroupHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		http.Error(w, "", http.StatusNotFound)
 		return
 	}
-	responsewriters.WriteObjectNegotiated(r.codecs, negotiation.DefaultEndpointRestrictions, schema.GroupVersion{}, w, req, http.StatusOK, discoveryGroup)
+	responsewriters.WriteObjectNegotiated(r.codecs, negotiation.DefaultEndpointRestrictions, schema.GroupVersion{}, w, req, http.StatusOK, discoveryGroup, false)
 }