From 8748813a61cfd4fb9583c9f5459b2ad30e23305e Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Fri, 26 May 2023 01:35:28 +0000 Subject: [PATCH] Use distinct clients for supervisor, deploy, and helm controllers Signed-off-by: Brad Davidson --- pkg/server/context.go | 21 ++----------- pkg/server/server.go | 71 +++++++++++++++++++++++++++++++------------ pkg/util/client.go | 17 +++++++++++ 3 files changed, 71 insertions(+), 38 deletions(-) diff --git a/pkg/server/context.go b/pkg/server/context.go index 0ad06230dc54..c8ecc2f6a911 100644 --- a/pkg/server/context.go +++ b/pkg/server/context.go @@ -2,26 +2,20 @@ package server import ( "context" - "fmt" - "os" - "runtime" helmcrd "github.com/k3s-io/helm-controller/pkg/crd" "github.com/k3s-io/helm-controller/pkg/generated/controllers/helm.cattle.io" addoncrd "github.com/k3s-io/k3s/pkg/crd" - "github.com/k3s-io/k3s/pkg/deploy" "github.com/k3s-io/k3s/pkg/generated/controllers/k3s.cattle.io" + "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/k3s/pkg/version" "github.com/pkg/errors" - "github.com/rancher/wrangler/pkg/apply" "github.com/rancher/wrangler/pkg/crd" "github.com/rancher/wrangler/pkg/generated/controllers/apps" "github.com/rancher/wrangler/pkg/generated/controllers/batch" "github.com/rancher/wrangler/pkg/generated/controllers/core" "github.com/rancher/wrangler/pkg/generated/controllers/rbac" "github.com/rancher/wrangler/pkg/start" - "github.com/sirupsen/logrus" - "k8s.io/apimachinery/pkg/apis/meta/v1/validation" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -35,7 +29,6 @@ type Context struct { Auth *rbac.Factory Core *core.Factory K8s kubernetes.Interface - Apply apply.Apply } func (c *Context) Start(ctx context.Context) error { @@ -47,16 +40,7 @@ func NewContext(ctx context.Context, cfg string) (*Context, error) { if err != nil { return nil, err } - - // Construct a custom user-agent string for the apply client used by the deploy controller - // so that we can track which node's deploy controller most recently modified a resource. - nodeName := os.Getenv("NODE_NAME") - managerName := deploy.ControllerName + "@" + nodeName - if nodeName == "" || len(managerName) > validation.FieldManagerMaxLength { - logrus.Warn("Deploy controller node name is empty or too long, and will not be tracked via server side apply field management") - managerName = deploy.ControllerName - } - restConfig.UserAgent = fmt.Sprintf("%s/%s (%s/%s) %s/%s", managerName, version.Version, runtime.GOOS, runtime.GOARCH, version.Program, version.GitCommit) + restConfig.UserAgent = util.GetUserAgent(version.Program + "-supervisor") if err := crds(ctx, restConfig); err != nil { return nil, errors.Wrap(err, "failed to register CRDs") @@ -74,7 +58,6 @@ func NewContext(ctx context.Context, cfg string) (*Context, error) { Apps: apps.NewFactoryFromConfigOrDie(restConfig), Batch: batch.NewFactoryFromConfigOrDie(restConfig), Core: core.NewFactoryFromConfigOrDie(restConfig), - Apply: apply.New(k8s, apply.NewClientFactory(restConfig)).WithDynamicLookup(), }, nil } diff --git a/pkg/server/server.go b/pkg/server/server.go index 018edc425b44..02cf6d7d0810 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -12,7 +12,7 @@ import ( "sync" "time" - helm "github.com/k3s-io/helm-controller/pkg/controllers/chart" + helmchart "github.com/k3s-io/helm-controller/pkg/controllers/chart" helmcommon "github.com/k3s-io/helm-controller/pkg/controllers/common" "github.com/k3s-io/k3s/pkg/cli/cmds" "github.com/k3s-io/k3s/pkg/clientaccess" @@ -28,12 +28,15 @@ import ( "github.com/k3s-io/k3s/pkg/util" "github.com/k3s-io/k3s/pkg/version" "github.com/pkg/errors" + "github.com/rancher/wrangler/pkg/apply" v1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1" "github.com/rancher/wrangler/pkg/leader" "github.com/rancher/wrangler/pkg/resolvehome" "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" ) const ( @@ -205,26 +208,42 @@ func coreControllers(ctx context.Context, sc *Context, config *Config) error { // apply SystemDefaultRegistry setting to Helm before starting controllers if config.ControlConfig.SystemDefaultRegistry != "" { - helm.DefaultJobImage = config.ControlConfig.SystemDefaultRegistry + "/" + helm.DefaultJobImage + helmchart.DefaultJobImage = config.ControlConfig.SystemDefaultRegistry + "/" + helmchart.DefaultJobImage } if !config.ControlConfig.DisableHelmController { - helm.Register(ctx, + restConfig, err := clientcmd.BuildConfigFromFlags("", config.ControlConfig.Runtime.KubeConfigAdmin) + if err != nil { + return err + } + restConfig.UserAgent = util.GetUserAgent(helmcommon.Name) + + k8s, err := clientset.NewForConfig(restConfig) + if err != nil { + return err + } + + apply := apply.New(k8s, apply.NewClientFactory(restConfig)).WithDynamicLookup() + helm := sc.Helm.WithAgent(restConfig.UserAgent) + batch := sc.Batch.WithAgent(restConfig.UserAgent) + auth := sc.Auth.WithAgent(restConfig.UserAgent) + core := sc.Core.WithAgent(restConfig.UserAgent) + helmchart.Register(ctx, metav1.NamespaceAll, helmcommon.Name, - sc.K8s, - sc.Apply, - util.BuildControllerEventRecorder(sc.K8s, helmcommon.Name, metav1.NamespaceAll), - sc.Helm.Helm().V1().HelmChart(), - sc.Helm.Helm().V1().HelmChart().Cache(), - sc.Helm.Helm().V1().HelmChartConfig(), - sc.Helm.Helm().V1().HelmChartConfig().Cache(), - sc.Batch.Batch().V1().Job(), - sc.Batch.Batch().V1().Job().Cache(), - sc.Auth.Rbac().V1().ClusterRoleBinding(), - sc.Core.Core().V1().ServiceAccount(), - sc.Core.Core().V1().ConfigMap(), - sc.Core.Core().V1().Secret()) + k8s, + apply, + util.BuildControllerEventRecorder(k8s, helmcommon.Name, metav1.NamespaceAll), + helm.V1().HelmChart(), + helm.V1().HelmChart().Cache(), + helm.V1().HelmChartConfig(), + helm.V1().HelmChartConfig().Cache(), + batch.V1().Job(), + batch.V1().Job().Cache(), + auth.V1().ClusterRoleBinding(), + core.V1().ServiceAccount(), + core.V1().ConfigMap(), + core.V1().Secret()) } if config.ControlConfig.EncryptSecrets { @@ -274,10 +293,24 @@ func stageFiles(ctx context.Context, sc *Context, controlConfig *config.Control) return err } + restConfig, err := clientcmd.BuildConfigFromFlags("", controlConfig.Runtime.KubeConfigAdmin) + if err != nil { + return err + } + restConfig.UserAgent = util.GetUserAgent("deploy") + + k8s, err := clientset.NewForConfig(restConfig) + if err != nil { + return err + } + + apply := apply.New(k8s, apply.NewClientFactory(restConfig)).WithDynamicLookup() + k3s := sc.K3s.WithAgent(restConfig.UserAgent) + return deploy.WatchFiles(ctx, - sc.K8s, - sc.Apply, - sc.K3s.K3s().V1().Addon(), + k8s, + apply, + k3s.V1().Addon(), controlConfig.Disables, dataDir) } diff --git a/pkg/util/client.go b/pkg/util/client.go index 6c6fefd00060..561a5cbc0817 100644 --- a/pkg/util/client.go +++ b/pkg/util/client.go @@ -1,9 +1,15 @@ package util import ( + "fmt" + "os" + "runtime" "strings" "github.com/k3s-io/k3s/pkg/datadir" + "github.com/k3s-io/k3s/pkg/version" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/apis/meta/v1/validation" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" ) @@ -30,6 +36,17 @@ func GetClientSet(file string) (clientset.Interface, error) { return clientset.NewForConfig(restConfig) } +// GetUserAgent builds a complete UserAgent string for a given controller, including the node name if possible. +func GetUserAgent(controllerName string) string { + nodeName := os.Getenv("NODE_NAME") + managerName := controllerName + "@" + nodeName + if nodeName == "" || len(managerName) > validation.FieldManagerMaxLength { + logrus.Warnf("%s controller node name is empty or too long, and will not be tracked via server side apply field management", controllerName) + managerName = controllerName + } + return fmt.Sprintf("%s/%s (%s/%s) %s/%s", managerName, version.Version, runtime.GOOS, runtime.GOARCH, version.Program, version.GitCommit) +} + // SplitStringSlice is a helper function to handle StringSliceFlag containing multiple values // By default, StringSliceFlag only supports repeated values, not multiple values // e.g. --foo="bar,car" --foo=baz will result in []string{"bar", "car". "baz"}