From a265194c7acb75e4810ff675bf08c4960e8d266e Mon Sep 17 00:00:00 2001 From: Zhuzhenghao Date: Wed, 10 May 2023 15:01:28 +0800 Subject: [PATCH] add flag --disable-qps-limits --- .../v1alpha1/kwokctl_configuration_types.go | 4 ++ .../config/v1alpha1/zz_generated.deepcopy.go | 5 ++ .../config/v1alpha1/zz_generated.defaults.go | 4 ++ .../kwokctl_configuration_types.go | 3 ++ .../zz_generated.conversion.go | 6 +++ pkg/consts/consts.go | 3 ++ pkg/kwokctl/cmd/create/cluster/cluster.go | 1 + pkg/kwokctl/components/kube_apiserver.go | 15 ++++++ .../components/kube_controller_manager.go | 9 ++++ pkg/kwokctl/components/kube_scheduler.go | 9 ++++ pkg/kwokctl/runtime/binary/cluster.go | 3 ++ pkg/kwokctl/runtime/compose/cluster.go | 3 ++ pkg/kwokctl/runtime/kind/cluster.go | 8 +++ pkg/kwokctl/runtime/kind/kind.go | 49 ++++++++++++++++++- site/content/en/docs/generated/apis.md | 11 +++++ .../docs/generated/kwokctl_create_cluster.md | 1 + test/kwokctl/suite.sh | 1 + 17 files changed, 134 insertions(+), 1 deletion(-) diff --git a/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go b/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go index 32bccb012..71dcb2b14 100644 --- a/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go +++ b/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go @@ -330,6 +330,10 @@ type KwokctlConfigurationOptions struct { // KubeApiserverCertSANs sets extra Subject Alternative Names for the API Server signing cert. KubeApiserverCertSANs []string `json:"kubeApiserverCertSANs,omitempty"` + + // DisableQPSLimits specifies whether to disable QPS limits for components. + // +default=false + DisableQPSLimits *bool `json:"disableQPSLimits,omitempty"` } // Component is a component of the cluster. diff --git a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go index fa370f1e0..fa944a3d6 100644 --- a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go @@ -285,6 +285,11 @@ func (in *KwokctlConfigurationOptions) DeepCopyInto(out *KwokctlConfigurationOpt *out = make([]string, len(*in)) copy(*out, *in) } + if in.DisableQPSLimits != nil { + in, out := &in.DisableQPSLimits, &out.DisableQPSLimits + *out = new(bool) + **out = **in + } return } diff --git a/pkg/apis/config/v1alpha1/zz_generated.defaults.go b/pkg/apis/config/v1alpha1/zz_generated.defaults.go index b65cffd55..2f8e5b321 100644 --- a/pkg/apis/config/v1alpha1/zz_generated.defaults.go +++ b/pkg/apis/config/v1alpha1/zz_generated.defaults.go @@ -97,6 +97,10 @@ func SetObjectDefaults_KwokctlConfiguration(in *KwokctlConfiguration) { if in.Options.BindAddress == "" { in.Options.BindAddress = "0.0.0.0" } + if in.Options.DisableQPSLimits == nil { + var ptrVar1 bool = false + in.Options.DisableQPSLimits = &ptrVar1 + } for i := range in.Components { a := &in.Components[i] for j := range a.Ports { diff --git a/pkg/apis/internalversion/kwokctl_configuration_types.go b/pkg/apis/internalversion/kwokctl_configuration_types.go index efa79ef28..a7c3cc5db 100644 --- a/pkg/apis/internalversion/kwokctl_configuration_types.go +++ b/pkg/apis/internalversion/kwokctl_configuration_types.go @@ -212,6 +212,9 @@ type KwokctlConfigurationOptions struct { // KubeApiserverCertSANs sets extra Subject Alternative Names for the API Server signing cert. KubeApiserverCertSANs []string + + // DisableQPSLimits specifies whether to disable QPS limits for components. + DisableQPSLimits bool } // Component is a component of the cluster. diff --git a/pkg/apis/internalversion/zz_generated.conversion.go b/pkg/apis/internalversion/zz_generated.conversion.go index 355dac16b..56cc3ab47 100644 --- a/pkg/apis/internalversion/zz_generated.conversion.go +++ b/pkg/apis/internalversion/zz_generated.conversion.go @@ -1344,6 +1344,9 @@ func autoConvert_internalversion_KwokctlConfigurationOptions_To_v1alpha1_Kwokctl out.NodeLeaseDurationSeconds = in.NodeLeaseDurationSeconds out.BindAddress = in.BindAddress out.KubeApiserverCertSANs = *(*[]string)(unsafe.Pointer(&in.KubeApiserverCertSANs)) + if err := v1.Convert_bool_To_Pointer_bool(&in.DisableQPSLimits, &out.DisableQPSLimits, s); err != nil { + return err + } return nil } @@ -1428,6 +1431,9 @@ func autoConvert_v1alpha1_KwokctlConfigurationOptions_To_internalversion_Kwokctl out.NodeLeaseDurationSeconds = in.NodeLeaseDurationSeconds out.BindAddress = in.BindAddress out.KubeApiserverCertSANs = *(*[]string)(unsafe.Pointer(&in.KubeApiserverCertSANs)) + if err := v1.Convert_Pointer_bool_To_bool(&in.DisableQPSLimits, &out.DisableQPSLimits, s); err != nil { + return err + } return nil } diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index fa247946f..1fa511c59 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -51,6 +51,9 @@ var ( PrometheusImagePrefix = "docker.io/prom" KindNodeImagePrefix = "docker.io/kindest" + + DefaultUnlimitedQPS = 5000.0 + DefaultUnlimitedBurst = 10000 ) // The following runtime is provided. diff --git a/pkg/kwokctl/cmd/create/cluster/cluster.go b/pkg/kwokctl/cmd/create/cluster/cluster.go index 02d887088..9146aebda 100644 --- a/pkg/kwokctl/cmd/create/cluster/cluster.go +++ b/pkg/kwokctl/cmd/create/cluster/cluster.go @@ -121,6 +121,7 @@ func NewCommand(ctx context.Context) *cobra.Command { cmd.Flags().DurationVar(&flags.Timeout, "timeout", 0, "Timeout for waiting for the cluster to be created") cmd.Flags().DurationVar(&flags.Wait, "wait", 0, "Wait for the cluster to be ready") cmd.Flags().StringVar(&flags.Kubeconfig, "kubeconfig", flags.Kubeconfig, "The path to the kubeconfig file will be added to the newly created cluster and set to current-context") + cmd.Flags().BoolVar(&flags.Options.DisableQPSLimits, "disable-qps-limits", flags.Options.DisableQPSLimits, "Disable QPS limits for components") return cmd } diff --git a/pkg/kwokctl/components/kube_apiserver.go b/pkg/kwokctl/components/kube_apiserver.go index dd8cb7a82..97e4e7854 100644 --- a/pkg/kwokctl/components/kube_apiserver.go +++ b/pkg/kwokctl/components/kube_apiserver.go @@ -46,6 +46,7 @@ type BuildKubeApiserverComponentConfig struct { AdminCertPath string AdminKeyPath string Verbosity log.Level + DisableQPSLimits bool ExtraArgs []internalversion.ExtraArgs ExtraVolumes []internalversion.Volume } @@ -84,6 +85,20 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon ) } + if conf.DisableQPSLimits { + kubeApiserverArgs = append(kubeApiserverArgs, + "--max-requests-inflight=0", + "--max-mutating-requests-inflight=0", + ) + + // FeatureGate APIPriorityAndFairness is not available before 1.17.0 + if conf.Version.GE(version.NewVersion(1, 18, 0)) { + kubeApiserverArgs = append(kubeApiserverArgs, + "--enable-priority-and-fairness=false", + ) + } + } + var ports []internalversion.Port var volumes []internalversion.Volume volumes = append(volumes, conf.ExtraVolumes...) diff --git a/pkg/kwokctl/components/kube_controller_manager.go b/pkg/kwokctl/components/kube_controller_manager.go index 283744b41..02dcafda5 100644 --- a/pkg/kwokctl/components/kube_controller_manager.go +++ b/pkg/kwokctl/components/kube_controller_manager.go @@ -20,6 +20,7 @@ import ( "time" "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/format" "sigs.k8s.io/kwok/pkg/utils/version" @@ -43,6 +44,7 @@ type BuildKubeControllerManagerComponentConfig struct { NodeMonitorPeriodMilliseconds int64 NodeMonitorGracePeriodMilliseconds int64 Verbosity log.Level + DisableQPSLimits bool ExtraArgs []internalversion.ExtraArgs ExtraVolumes []internalversion.Volume } @@ -180,6 +182,13 @@ func BuildKubeControllerManagerComponent(conf BuildKubeControllerManagerComponen } } + if conf.DisableQPSLimits { + kubeControllerManagerArgs = append(kubeControllerManagerArgs, + "--kube-api-qps="+format.String(consts.DefaultUnlimitedQPS), + "--kube-api-burst="+format.String(consts.DefaultUnlimitedBurst), + ) + } + if conf.Verbosity != log.LevelInfo { kubeControllerManagerArgs = append(kubeControllerManagerArgs, "--v="+format.String(log.ToKlogLevel(conf.Verbosity))) } diff --git a/pkg/kwokctl/components/kube_scheduler.go b/pkg/kwokctl/components/kube_scheduler.go index d687cc1f5..6a72521ea 100644 --- a/pkg/kwokctl/components/kube_scheduler.go +++ b/pkg/kwokctl/components/kube_scheduler.go @@ -18,6 +18,7 @@ package components import ( "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/format" "sigs.k8s.io/kwok/pkg/utils/version" @@ -39,6 +40,7 @@ type BuildKubeSchedulerComponentConfig struct { KubeconfigPath string KubeFeatureGates string Verbosity log.Level + DisableQPSLimits bool ExtraArgs []internalversion.ExtraArgs ExtraVolumes []internalversion.Volume } @@ -170,6 +172,13 @@ func BuildKubeSchedulerComponent(conf BuildKubeSchedulerComponentConfig) (compon // ) } + if conf.DisableQPSLimits { + kubeSchedulerArgs = append(kubeSchedulerArgs, + "--kube-api-qps="+format.String(consts.DefaultUnlimitedQPS), + "--kube-api-burst="+format.String(consts.DefaultUnlimitedBurst), + ) + } + if conf.Verbosity != log.LevelInfo { kubeSchedulerArgs = append(kubeSchedulerArgs, "--v="+format.String(log.ToKlogLevel(conf.Verbosity))) } diff --git a/pkg/kwokctl/runtime/binary/cluster.go b/pkg/kwokctl/runtime/binary/cluster.go index 5bc4b84a5..6cec03b7b 100644 --- a/pkg/kwokctl/runtime/binary/cluster.go +++ b/pkg/kwokctl/runtime/binary/cluster.go @@ -296,6 +296,7 @@ func (c *Cluster) Install(ctx context.Context) error { AdminCertPath: adminCertPath, AdminKeyPath: adminKeyPath, Verbosity: verbosity, + DisableQPSLimits: conf.DisableQPSLimits, ExtraArgs: kubeApiserverComponentPatches.ExtraArgs, ExtraVolumes: kubeApiserverComponentPatches.ExtraVolumes, }) @@ -335,6 +336,7 @@ func (c *Cluster) Install(ctx context.Context) error { NodeMonitorPeriodMilliseconds: conf.KubeControllerManagerNodeMonitorPeriodMilliseconds, NodeMonitorGracePeriodMilliseconds: conf.KubeControllerManagerNodeMonitorGracePeriodMilliseconds, Verbosity: verbosity, + DisableQPSLimits: conf.DisableQPSLimits, ExtraArgs: kubeControllerManagerPatches.ExtraArgs, ExtraVolumes: kubeControllerManagerPatches.ExtraVolumes, }) @@ -382,6 +384,7 @@ func (c *Cluster) Install(ctx context.Context) error { KubeconfigPath: kubeconfigPath, KubeFeatureGates: conf.KubeFeatureGates, Verbosity: verbosity, + DisableQPSLimits: conf.DisableQPSLimits, ExtraArgs: kubeSchedulerComponentPatches.ExtraArgs, ExtraVolumes: kubeSchedulerComponentPatches.ExtraVolumes, }) diff --git a/pkg/kwokctl/runtime/compose/cluster.go b/pkg/kwokctl/runtime/compose/cluster.go index ee707aa57..8f6d87924 100644 --- a/pkg/kwokctl/runtime/compose/cluster.go +++ b/pkg/kwokctl/runtime/compose/cluster.go @@ -322,6 +322,7 @@ func (c *Cluster) Install(ctx context.Context) error { EtcdPort: conf.EtcdPort, EtcdAddress: c.Name() + "-etcd", Verbosity: verbosity, + DisableQPSLimits: conf.DisableQPSLimits, ExtraArgs: kubeApiserverComponentPatches.ExtraArgs, ExtraVolumes: kubeApiserverComponentPatches.ExtraVolumes, }) @@ -356,6 +357,7 @@ func (c *Cluster) Install(ctx context.Context) error { KubeconfigPath: inClusterOnHostKubeconfigPath, KubeFeatureGates: conf.KubeFeatureGates, Verbosity: verbosity, + DisableQPSLimits: conf.DisableQPSLimits, NodeMonitorPeriodMilliseconds: conf.KubeControllerManagerNodeMonitorPeriodMilliseconds, NodeMonitorGracePeriodMilliseconds: conf.KubeControllerManagerNodeMonitorGracePeriodMilliseconds, ExtraArgs: kubeControllerManagerComponentPatches.ExtraArgs, @@ -402,6 +404,7 @@ func (c *Cluster) Install(ctx context.Context) error { KubeconfigPath: inClusterOnHostKubeconfigPath, KubeFeatureGates: conf.KubeFeatureGates, Verbosity: verbosity, + DisableQPSLimits: conf.DisableQPSLimits, ExtraArgs: kubeSchedulerComponentPatches.ExtraArgs, ExtraVolumes: kubeSchedulerComponentPatches.ExtraVolumes, }) diff --git a/pkg/kwokctl/runtime/kind/cluster.go b/pkg/kwokctl/runtime/kind/cluster.go index 0a812e532..f992d5f57 100644 --- a/pkg/kwokctl/runtime/kind/cluster.go +++ b/pkg/kwokctl/runtime/kind/cluster.go @@ -41,6 +41,7 @@ import ( "sigs.k8s.io/kwok/pkg/utils/net" "sigs.k8s.io/kwok/pkg/utils/path" "sigs.k8s.io/kwok/pkg/utils/slices" + "sigs.k8s.io/kwok/pkg/utils/version" "sigs.k8s.io/kwok/pkg/utils/wait" ) @@ -132,6 +133,11 @@ func (c *Cluster) Install(ctx context.Context) error { configPath := c.GetWorkdirPath(runtime.ConfigName) + kubeVersion, err := version.ParseVersion(conf.KubeVersion) + if err != nil { + return err + } + etcdComponentPatches := runtime.GetComponentPatches(config, "etcd") kubeApiserverComponentPatches := runtime.GetComponentPatches(config, "kube-apiserver") kubeSchedulerComponentPatches := runtime.GetComponentPatches(config, "kube-scheduler") @@ -162,6 +168,8 @@ func (c *Cluster) Install(ctx context.Context) error { ControllerManagerExtraArgs: kubeControllerManagerComponentPatches.ExtraArgs, ControllerManagerExtraVolumes: kubeControllerManagerComponentPatches.ExtraVolumes, KwokControllerExtraVolumes: kwokControllerExtraVolumes, + DisableQPSLimits: conf.DisableQPSLimits, + KubeVersion: kubeVersion, }) if err != nil { return err diff --git a/pkg/kwokctl/runtime/kind/kind.go b/pkg/kwokctl/runtime/kind/kind.go index 7660864be..1f917a9e3 100644 --- a/pkg/kwokctl/runtime/kind/kind.go +++ b/pkg/kwokctl/runtime/kind/kind.go @@ -22,9 +22,11 @@ import ( "text/template" "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/kwokctl/runtime" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/format" + "sigs.k8s.io/kwok/pkg/utils/version" _ "embed" ) @@ -137,6 +139,49 @@ func expendExtrasForBuildKind(conf BuildKindConfig) BuildKindConfig { }, ) } + + if conf.DisableQPSLimits { + conf.ApiserverExtraArgs = append(conf.ApiserverExtraArgs, + internalversion.ExtraArgs{ + Key: "max-requests-inflight", + Value: "0", + }, + internalversion.ExtraArgs{ + Key: "max-mutating-requests-inflight", + Value: "0", + }, + ) + + // FeatureGate APIPriorityAndFairness is not available before 1.17.0 + if conf.KubeVersion.GE(version.NewVersion(1, 18, 0)) { + conf.ApiserverExtraArgs = append(conf.ApiserverExtraArgs, + internalversion.ExtraArgs{ + Key: "enable-priority-and-fairness", + Value: "false", + }, + ) + } + conf.ControllerManagerExtraArgs = append(conf.ControllerManagerExtraArgs, + internalversion.ExtraArgs{ + Key: "kube-api-qps", + Value: format.String(consts.DefaultUnlimitedQPS), + }, + internalversion.ExtraArgs{ + Key: "kube-api-burst", + Value: format.String(consts.DefaultUnlimitedBurst), + }, + ) + conf.SchedulerExtraArgs = append(conf.SchedulerExtraArgs, + internalversion.ExtraArgs{ + Key: "kube-api-qps", + Value: format.String(consts.DefaultUnlimitedQPS), + }, + internalversion.ExtraArgs{ + Key: "kube-api-burst", + Value: format.String(consts.DefaultUnlimitedBurst), + }, + ) + } return conf } @@ -203,5 +248,7 @@ type BuildKindConfig struct { ControllerManagerExtraVolumes []internalversion.Volume KwokControllerExtraVolumes []internalversion.Volume - BindAddress string + BindAddress string + DisableQPSLimits bool + KubeVersion version.Version } diff --git a/site/content/en/docs/generated/apis.md b/site/content/en/docs/generated/apis.md index 84e6b1c96..4f7e420e3 100644 --- a/site/content/en/docs/generated/apis.md +++ b/site/content/en/docs/generated/apis.md @@ -2442,6 +2442,17 @@ string

KubeApiserverCertSANs sets extra Subject Alternative Names for the API Server signing cert.

+ + +disableQPSLimits + +bool + + + +

DisableQPSLimits specifies whether to disable QPS limits for components.

+ +

diff --git a/site/content/en/docs/generated/kwokctl_create_cluster.md b/site/content/en/docs/generated/kwokctl_create_cluster.md index bff29fc2b..84584dbe8 100644 --- a/site/content/en/docs/generated/kwokctl_create_cluster.md +++ b/site/content/en/docs/generated/kwokctl_create_cluster.md @@ -12,6 +12,7 @@ kwokctl create cluster [flags] --controller-port uint32 Port of kwok-controller given to the host --disable-kube-controller-manager Disable the kube-controller-manager --disable-kube-scheduler Disable the kube-scheduler + --disable-qps-limits Disable QPS limits for components --etcd-binary string Binary of etcd, only for binary runtime --etcd-binary-tar string Tar of etcd, if --etcd-binary is set, this is ignored, only for binary runtime (default "https://github.com/etcd-io/etcd/releases/download/v3.5.7/etcd-v3.5.7-linux-amd64.tar.gz") diff --git a/test/kwokctl/suite.sh b/test/kwokctl/suite.sh index e98d3a5c6..5beb846fe 100644 --- a/test/kwokctl/suite.sh +++ b/test/kwokctl/suite.sh @@ -53,6 +53,7 @@ function create_cluster() { --timeout 30m \ --wait 30m \ --quiet-pull \ + --disable-qps-limits \ "$@"; then echo "Error: Cluster ${name} creation failed" show_all