Skip to content

Commit

Permalink
Use uncached client and partial metadata for secret and configmaps
Browse files Browse the repository at this point in the history 
These changes allow the current stable Cluster API release to reduce its
memory footprint by a large margin.

Currently, we have multiple controllers that are either watching (like
ClusterResourceSet) or querying (get/list) corev1.Secret and
corev1.ConfigMap resources.

When these kinds go through the case, all of the objects in the cluster
end up being cached, not just the ones we're interested in.

In production environments, there might be a large number of ConfigMap
or Secret resources that we end up caching and watching for little gain.

Signed-off-by: Vince Prignano <vincepri@vmware.com>
  • Loading branch information
@vincepri
vincepri committed Dec 15, 2020
1 parent ffff2a3 commit 66be2bb
Show file tree
Hide file tree
Showing 13 changed files with 57 additions and 48 deletions.
8 changes: 7 additions & 1 deletion bootstrap/kubeadm/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"time"

"github.com/spf13/pflag"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/klog"
Expand All @@ -37,6 +38,7 @@ import (
expv1 "sigs.k8s.io/cluster-api/exp/api/v1alpha4"
"sigs.k8s.io/cluster-api/feature"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
// +kubebuilder:scaffold:imports
)
Expand Down Expand Up @@ -132,7 +134,11 @@ func main() {
RetryPeriod: &leaderElectionRetryPeriod,
Namespace: watchNamespace,
SyncPeriod: &syncPeriod,
Port: webhookPort,
ClientDisableCacheFor: []client.Object{
&corev1.ConfigMap{},
&corev1.Secret{},
},
Port: webhookPort,
})
if err != nil {
setupLog.Error(err, "unable to start manager")
Expand Down
34 changes: 12 additions & 22 deletions controllers/remote/cluster_cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (

"github.com/go-logr/logr"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"
Expand Down Expand Up @@ -78,28 +79,14 @@ func (t *ClusterCacheTracker) GetClient(ctx context.Context, cluster client.Obje
return nil, err
}

return accessor.delegatingClient, nil
}

// GetLiveClient returns a live client (talks to the api-server directly) for the given cluster.
func (t *ClusterCacheTracker) GetLiveClient(ctx context.Context, cluster client.ObjectKey) (client.Client, error) {
t.lock.Lock()
defer t.lock.Unlock()

accessor, err := t.getClusterAccessorLH(ctx, cluster)
if err != nil {
return nil, err
}

return accessor.liveClient, nil
return accessor.client, nil
}

// clusterAccessor represents the combination of a delegating client, live client (direct to api-server), cache, and watches for a remote cluster.
type clusterAccessor struct {
cache *stoppableCache
delegatingClient client.Client
liveClient client.Client
watches sets.String
cache *stoppableCache
client client.Client
watches sets.String
}

// clusterAccessorExists returns true if a clusterAccessor exists for cluster.
Expand Down Expand Up @@ -180,16 +167,19 @@ func (t *ClusterCacheTracker) newClusterAccessor(ctx context.Context, cluster cl
delegatingClient, err := client.NewDelegatingClient(client.NewDelegatingClientInput{
CacheReader: cache,
Client: c,
UncachedObjects: []client.Object{
&corev1.ConfigMap{},
&corev1.Secret{},
},
})
if err != nil {
return nil, err
}

return &clusterAccessor{
cache: cache,
delegatingClient: delegatingClient,
liveClient: c,
watches: sets.NewString(),
cache: cache,
client: delegatingClient,
watches: sets.NewString(),
}, nil
}

Expand Down
6 changes: 3 additions & 3 deletions controllers/remote/cluster_cache_fake.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,9 @@ func NewTestClusterCacheTracker(log logr.Logger, cl client.Client, scheme *runti

testCacheTracker.clusterAccessors[objKey] = &clusterAccessor{

cache: nil,
delegatingClient: delegatingClient,
watches: sets.NewString(watchObjects...),
cache: nil,
client: delegatingClient,
watches: sets.NewString(watchObjects...),
}
return testCacheTracker
}
4 changes: 0 additions & 4 deletions controllers/remote/cluster_cache_reconciler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,10 +73,6 @@ var _ = Describe("ClusterCache Reconciler suite", func() {
By("Creating a clusterAccessor for the cluster")
_, err := cct.GetClient(ctx, testClusterKey)
Expect(err).To(BeNil())

By("Retrieving a live client for the cluster")
_, err = cct.GetLiveClient(ctx, testClusterKey)
Expect(err).To(BeNil())
}

BeforeEach(func() {
Expand Down
8 changes: 7 additions & 1 deletion controlplane/kubeadm/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"time"

"github.com/spf13/pflag"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/klog"
Expand All @@ -36,6 +37,7 @@ import (
kcpv1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha4"
kubeadmcontrolplanecontrollers "sigs.k8s.io/cluster-api/controlplane/kubeadm/controllers"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
// +kubebuilder:scaffold:imports
)
Expand Down Expand Up @@ -126,7 +128,11 @@ func main() {
RetryPeriod: &leaderElectionRetryPeriod,
Namespace: watchNamespace,
SyncPeriod: &syncPeriod,
Port: webhookPort,
ClientDisableCacheFor: []client.Object{
&corev1.ConfigMap{},
&corev1.Secret{},
},
Port: webhookPort,
})
if err != nil {
setupLog.Error(err, "unable to start manager")
Expand Down
6 changes: 4 additions & 2 deletions exp/addons/controllers/clusterresourceset_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ type ClusterResourceSetReconciler struct {
}

func (r *ClusterResourceSetReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error {
_, err := ctrl.NewControllerManagedBy(mgr).
err := ctrl.NewControllerManagedBy(mgr).
For(&addonsv1.ClusterResourceSet{}).
Watches(
&source.Kind{Type: &clusterv1.Cluster{}},
Expand All @@ -75,20 +75,22 @@ func (r *ClusterResourceSetReconciler) SetupWithManager(ctx context.Context, mgr
Watches(
&source.Kind{Type: &corev1.ConfigMap{}},
handler.EnqueueRequestsFromMapFunc(r.resourceToClusterResourceSet),
builder.OnlyMetadata,
builder.WithPredicates(
resourcepredicates.ResourceCreate(ctrl.LoggerFrom(ctx)),
),
).
Watches(
&source.Kind{Type: &corev1.Secret{}},
handler.EnqueueRequestsFromMapFunc(r.resourceToClusterResourceSet),
builder.OnlyMetadata,
builder.WithPredicates(
resourcepredicates.AddonsSecretCreate(ctrl.LoggerFrom(ctx)),
),
).
WithOptions(options).
WithEventFilter(predicates.ResourceNotPaused(ctrl.LoggerFrom(ctx))).
Build(r)
Complete(r)

if err != nil {
return errors.Wrap(err, "failed setting up with a controller manager")
Expand Down
1 change: 1 addition & 0 deletions exp/addons/controllers/suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ var _ = BeforeSuite(func(done Done) {

By("starting the manager")
go func() {
defer GinkgoRecover()
Expect(testEnv.StartManager(ctx)).To(Succeed())
}()

Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ require (
k8s.io/component-base v0.19.2
k8s.io/klog v1.0.0
k8s.io/utils v0.0.0-20200912215256-4140de9c8800
sigs.k8s.io/controller-runtime v0.7.0-alpha.8
sigs.k8s.io/controller-runtime v0.7.1-0.20201215171748-096b2e07c091
sigs.k8s.io/kind v0.9.0
sigs.k8s.io/yaml v1.2.0
)
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -797,8 +797,8 @@ k8s.io/utils v0.0.0-20200912215256-4140de9c8800 h1:9ZNvfPvVIEsp/T1ez4GQuzCcCTEQW
k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0=
sigs.k8s.io/controller-runtime v0.7.0-alpha.8 h1:l8I2KO3xLuNaT0yPP6mtWLw5NuMG3dY2YiaQXEGApXg=
sigs.k8s.io/controller-runtime v0.7.0-alpha.8/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU=
sigs.k8s.io/controller-runtime v0.7.1-0.20201215171748-096b2e07c091 h1:tqrTDj7mJmM6TdpoM1rN2PzBRH9yzCReqKGMy4sp+f0=
sigs.k8s.io/controller-runtime v0.7.1-0.20201215171748-096b2e07c091/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU=
sigs.k8s.io/kind v0.9.0 h1:SoDlXq6pEc7dGagHULNRCCBYrLH6xOi7lqXTRXeAlg4=
sigs.k8s.io/kind v0.9.0/go.mod h1:cxKQWwmbtRDzQ+RNKnR6gZG6fjbeTtItp5cGf+ww+1Y=
sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
Expand Down
24 changes: 15 additions & 9 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"time"

"github.com/spf13/pflag"
corev1 "k8s.io/api/core/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
Expand All @@ -40,6 +41,7 @@ import (
expcontrollers "sigs.k8s.io/cluster-api/exp/controllers"
"sigs.k8s.io/cluster-api/feature"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/healthz"
// +kubebuilder:scaffold:imports
Expand Down Expand Up @@ -153,15 +155,19 @@ func main() {
}

mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
Scheme: scheme,
MetricsBindAddress: metricsBindAddr,
LeaderElection: enableLeaderElection,
LeaderElectionID: "controller-leader-election-capi",
LeaseDuration: &leaderElectionLeaseDuration,
RenewDeadline: &leaderElectionRenewDeadline,
RetryPeriod: &leaderElectionRetryPeriod,
Namespace: watchNamespace,
SyncPeriod: &syncPeriod,
Scheme: scheme,
MetricsBindAddress: metricsBindAddr,
LeaderElection: enableLeaderElection,
LeaderElectionID: "controller-leader-election-capi",
LeaseDuration: &leaderElectionLeaseDuration,
RenewDeadline: &leaderElectionRenewDeadline,
RetryPeriod: &leaderElectionRetryPeriod,
Namespace: watchNamespace,
SyncPeriod: &syncPeriod,
ClientDisableCacheFor: []client.Object{
&corev1.ConfigMap{},
&corev1.Secret{},
},
Port: webhookPort,
HealthProbeBindAddress: healthAddr,
})
Expand Down
2 changes: 2 additions & 0 deletions test/framework/convenience.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4"
bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1alpha4"
controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1alpha4"
addonsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1alpha3"
expv1 "sigs.k8s.io/cluster-api/exp/api/v1alpha4"
)

Expand All @@ -50,6 +51,7 @@ func TryAddDefaultSchemes(scheme *runtime.Scheme) {

// Add the experiments CAPI scheme.
_ = expv1.AddToScheme(scheme)
_ = addonsv1.AddToScheme(scheme)

// Add the kubeadm bootstrapper scheme.
_ = bootstrapv1.AddToScheme(scheme)
Expand Down
2 changes: 1 addition & 1 deletion test/infrastructure/docker/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ require (
k8s.io/klog v1.0.0
k8s.io/utils v0.0.0-20200912215256-4140de9c8800
sigs.k8s.io/cluster-api v0.3.3
sigs.k8s.io/controller-runtime v0.7.0-alpha.8
sigs.k8s.io/controller-runtime v0.7.1-0.20201215171748-096b2e07c091
sigs.k8s.io/kind v0.9.0
sigs.k8s.io/yaml v1.2.0
)
Expand Down
4 changes: 2 additions & 2 deletions test/infrastructure/docker/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -741,8 +741,8 @@ k8s.io/utils v0.0.0-20200912215256-4140de9c8800 h1:9ZNvfPvVIEsp/T1ez4GQuzCcCTEQW
k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0=
sigs.k8s.io/controller-runtime v0.7.0-alpha.8 h1:l8I2KO3xLuNaT0yPP6mtWLw5NuMG3dY2YiaQXEGApXg=
sigs.k8s.io/controller-runtime v0.7.0-alpha.8/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU=
sigs.k8s.io/controller-runtime v0.7.1-0.20201215171748-096b2e07c091 h1:tqrTDj7mJmM6TdpoM1rN2PzBRH9yzCReqKGMy4sp+f0=
sigs.k8s.io/controller-runtime v0.7.1-0.20201215171748-096b2e07c091/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU=
sigs.k8s.io/kind v0.9.0 h1:SoDlXq6pEc7dGagHULNRCCBYrLH6xOi7lqXTRXeAlg4=
sigs.k8s.io/kind v0.9.0/go.mod h1:cxKQWwmbtRDzQ+RNKnR6gZG6fjbeTtItp5cGf+ww+1Y=
sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
Expand Down

0 comments on commit 66be2bb

Please sign in to comment.