From 20a2a8dff79cb3ac1a49b8746983c84e529e56ef Mon Sep 17 00:00:00 2001 From: Zhuzhenghao Date: Wed, 10 May 2023 15:01:28 +0800 Subject: [PATCH] add flag --disable-qps-limits --- .../v1alpha1/kwokctl_configuration_types.go | 4 ++ .../config/v1alpha1/zz_generated.deepcopy.go | 5 ++ .../config/v1alpha1/zz_generated.defaults.go | 4 ++ .../kwokctl_configuration_types.go | 3 ++ .../zz_generated.conversion.go | 6 +++ pkg/consts/consts.go | 3 ++ pkg/kwokctl/cmd/create/cluster/cluster.go | 1 + pkg/kwokctl/components/kube_apiserver.go | 15 ++++++ .../components/kube_controller_manager.go | 10 ++++ pkg/kwokctl/components/kube_scheduler.go | 11 ++++ pkg/kwokctl/runtime/binary/cluster.go | 3 ++ pkg/kwokctl/runtime/compose/cluster.go | 3 ++ pkg/kwokctl/runtime/kind/cluster.go | 8 +++ pkg/kwokctl/runtime/kind/kind.go | 50 ++++++++++++++++++- site/content/en/docs/generated/apis.md | 11 ++++ .../docs/generated/kwokctl_create_cluster.md | 1 + test/kwokctl/suite.sh | 1 + 17 files changed, 138 insertions(+), 1 deletion(-) diff --git a/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go b/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go index b5b5d757e9..7aacd9d9f8 100644 --- a/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go +++ b/pkg/apis/config/v1alpha1/kwokctl_configuration_types.go @@ -324,6 +324,10 @@ type KwokctlConfigurationOptions struct { // BindAddress is the address to bind to. // +default="0.0.0.0" BindAddress string `json:"bindAddress,omitempty"` + + // DisableQPSLimits specifies whether to disable QPS limits for components. + // +default=false + DisableQPSLimits *bool `json:"disableQPSLimits,omitempty"` } // Component is a component of the cluster. diff --git a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go index f5af87b62c..cbf35decf4 100644 --- a/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha1/zz_generated.deepcopy.go @@ -280,6 +280,11 @@ func (in *KwokctlConfigurationOptions) DeepCopyInto(out *KwokctlConfigurationOpt *out = new(bool) **out = **in } + if in.DisableQPSLimits != nil { + in, out := &in.DisableQPSLimits, &out.DisableQPSLimits + *out = new(bool) + **out = **in + } return } diff --git a/pkg/apis/config/v1alpha1/zz_generated.defaults.go b/pkg/apis/config/v1alpha1/zz_generated.defaults.go index b65cffd550..2f8e5b3210 100644 --- a/pkg/apis/config/v1alpha1/zz_generated.defaults.go +++ b/pkg/apis/config/v1alpha1/zz_generated.defaults.go @@ -97,6 +97,10 @@ func SetObjectDefaults_KwokctlConfiguration(in *KwokctlConfiguration) { if in.Options.BindAddress == "" { in.Options.BindAddress = "0.0.0.0" } + if in.Options.DisableQPSLimits == nil { + var ptrVar1 bool = false + in.Options.DisableQPSLimits = &ptrVar1 + } for i := range in.Components { a := &in.Components[i] for j := range a.Ports { diff --git a/pkg/apis/internalversion/kwokctl_configuration_types.go b/pkg/apis/internalversion/kwokctl_configuration_types.go index cc5d3e058d..5b60fa074f 100644 --- a/pkg/apis/internalversion/kwokctl_configuration_types.go +++ b/pkg/apis/internalversion/kwokctl_configuration_types.go @@ -209,6 +209,9 @@ type KwokctlConfigurationOptions struct { // BindAddress is the address to bind to. BindAddress string + + // DisableQPSLimits specifies whether to disable QPS limits for components. + DisableQPSLimits bool } // Component is a component of the cluster. diff --git a/pkg/apis/internalversion/zz_generated.conversion.go b/pkg/apis/internalversion/zz_generated.conversion.go index b6518a4c34..f66dcc78ee 100644 --- a/pkg/apis/internalversion/zz_generated.conversion.go +++ b/pkg/apis/internalversion/zz_generated.conversion.go @@ -1343,6 +1343,9 @@ func autoConvert_internalversion_KwokctlConfigurationOptions_To_v1alpha1_Kwokctl out.NodeStatusUpdateFrequencyMilliseconds = in.NodeStatusUpdateFrequencyMilliseconds out.NodeLeaseDurationSeconds = in.NodeLeaseDurationSeconds out.BindAddress = in.BindAddress + if err := v1.Convert_bool_To_Pointer_bool(&in.DisableQPSLimits, &out.DisableQPSLimits, s); err != nil { + return err + } return nil } @@ -1426,6 +1429,9 @@ func autoConvert_v1alpha1_KwokctlConfigurationOptions_To_internalversion_Kwokctl out.NodeStatusUpdateFrequencyMilliseconds = in.NodeStatusUpdateFrequencyMilliseconds out.NodeLeaseDurationSeconds = in.NodeLeaseDurationSeconds out.BindAddress = in.BindAddress + if err := v1.Convert_Pointer_bool_To_bool(&in.DisableQPSLimits, &out.DisableQPSLimits, s); err != nil { + return err + } return nil } diff --git a/pkg/consts/consts.go b/pkg/consts/consts.go index fa247946f8..1fa511c594 100644 --- a/pkg/consts/consts.go +++ b/pkg/consts/consts.go @@ -51,6 +51,9 @@ var ( PrometheusImagePrefix = "docker.io/prom" KindNodeImagePrefix = "docker.io/kindest" + + DefaultUnlimitedQPS = 5000.0 + DefaultUnlimitedBurst = 10000 ) // The following runtime is provided. diff --git a/pkg/kwokctl/cmd/create/cluster/cluster.go b/pkg/kwokctl/cmd/create/cluster/cluster.go index aea8d45236..d315d99ced 100644 --- a/pkg/kwokctl/cmd/create/cluster/cluster.go +++ b/pkg/kwokctl/cmd/create/cluster/cluster.go @@ -119,6 +119,7 @@ func NewCommand(ctx context.Context) *cobra.Command { cmd.Flags().DurationVar(&flags.Timeout, "timeout", 0, "Timeout for waiting for the cluster to be created") cmd.Flags().DurationVar(&flags.Wait, "wait", 0, "Wait for the cluster to be ready") cmd.Flags().StringVar(&flags.Kubeconfig, "kubeconfig", flags.Kubeconfig, "The path to the kubeconfig file will be added to the newly created cluster and set to current-context") + cmd.Flags().BoolVar(&flags.Options.DisableQPSLimits, "disable-qps-limits", flags.Options.DisableQPSLimits, "Disable QPS limits for components") return cmd } diff --git a/pkg/kwokctl/components/kube_apiserver.go b/pkg/kwokctl/components/kube_apiserver.go index dd8cb7a824..97e4e78540 100644 --- a/pkg/kwokctl/components/kube_apiserver.go +++ b/pkg/kwokctl/components/kube_apiserver.go @@ -46,6 +46,7 @@ type BuildKubeApiserverComponentConfig struct { AdminCertPath string AdminKeyPath string Verbosity log.Level + DisableQPSLimits bool ExtraArgs []internalversion.ExtraArgs ExtraVolumes []internalversion.Volume } @@ -84,6 +85,20 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon ) } + if conf.DisableQPSLimits { + kubeApiserverArgs = append(kubeApiserverArgs, + "--max-requests-inflight=0", + "--max-mutating-requests-inflight=0", + ) + + // FeatureGate APIPriorityAndFairness is not available before 1.17.0 + if conf.Version.GE(version.NewVersion(1, 18, 0)) { + kubeApiserverArgs = append(kubeApiserverArgs, + "--enable-priority-and-fairness=false", + ) + } + } + var ports []internalversion.Port var volumes []internalversion.Volume volumes = append(volumes, conf.ExtraVolumes...) diff --git a/pkg/kwokctl/components/kube_controller_manager.go b/pkg/kwokctl/components/kube_controller_manager.go index bb1d520c5f..e63b9f16a6 100644 --- a/pkg/kwokctl/components/kube_controller_manager.go +++ b/pkg/kwokctl/components/kube_controller_manager.go @@ -17,9 +17,11 @@ limitations under the License. package components import ( + "strconv" "time" "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/format" "sigs.k8s.io/kwok/pkg/utils/version" @@ -43,6 +45,7 @@ type BuildKubeControllerManagerComponentConfig struct { NodeMonitorPeriodMilliseconds int64 NodeMonitorGracePeriodMilliseconds int64 Verbosity log.Level + DisableQPSLimits bool ExtraArgs []internalversion.ExtraArgs ExtraVolumes []internalversion.Volume } @@ -182,6 +185,13 @@ func BuildKubeControllerManagerComponent(conf BuildKubeControllerManagerComponen } } + if conf.DisableQPSLimits { + kubeControllerManagerArgs = append(kubeControllerManagerArgs, + "--kube-api-qps="+strconv.FormatFloat(consts.DefaultUnlimitedQPS, 'f', 1, 64), + "--kube-api-burst="+format.String(consts.DefaultUnlimitedBurst), + ) + } + if conf.Verbosity != log.LevelInfo { kubeControllerManagerArgs = append(kubeControllerManagerArgs, "--v="+format.String(log.ToKlogLevel(conf.Verbosity))) } diff --git a/pkg/kwokctl/components/kube_scheduler.go b/pkg/kwokctl/components/kube_scheduler.go index d8b0e165ea..06a4b6e79e 100644 --- a/pkg/kwokctl/components/kube_scheduler.go +++ b/pkg/kwokctl/components/kube_scheduler.go @@ -17,7 +17,10 @@ limitations under the License. package components import ( + "strconv" + "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/format" "sigs.k8s.io/kwok/pkg/utils/version" @@ -39,6 +42,7 @@ type BuildKubeSchedulerComponentConfig struct { KubeconfigPath string KubeFeatureGates string Verbosity log.Level + DisableQPSLimits bool ExtraArgs []internalversion.ExtraArgs ExtraVolumes []internalversion.Volume } @@ -165,6 +169,13 @@ func BuildKubeSchedulerComponent(conf BuildKubeSchedulerComponentConfig) (compon // ) } + if conf.DisableQPSLimits { + kubeSchedulerArgs = append(kubeSchedulerArgs, + "--kube-api-qps="+strconv.FormatFloat(consts.DefaultUnlimitedQPS, 'f', 1, 64), + "--kube-api-burst="+format.String(consts.DefaultUnlimitedBurst), + ) + } + if conf.Verbosity != log.LevelInfo { kubeSchedulerArgs = append(kubeSchedulerArgs, "--v="+format.String(log.ToKlogLevel(conf.Verbosity))) } diff --git a/pkg/kwokctl/runtime/binary/cluster.go b/pkg/kwokctl/runtime/binary/cluster.go index 870d7417c4..5ef8684173 100644 --- a/pkg/kwokctl/runtime/binary/cluster.go +++ b/pkg/kwokctl/runtime/binary/cluster.go @@ -284,6 +284,7 @@ func (c *Cluster) Install(ctx context.Context) error { AdminCertPath: adminCertPath, AdminKeyPath: adminKeyPath, Verbosity: verbosity, + DisableQPSLimits: conf.DisableQPSLimits, ExtraArgs: kubeApiserverComponentPatches.ExtraArgs, ExtraVolumes: kubeApiserverComponentPatches.ExtraVolumes, }) @@ -322,6 +323,7 @@ func (c *Cluster) Install(ctx context.Context) error { NodeMonitorPeriodMilliseconds: conf.KubeControllerManagerNodeMonitorPeriodMilliseconds, NodeMonitorGracePeriodMilliseconds: conf.KubeControllerManagerNodeMonitorGracePeriodMilliseconds, Verbosity: verbosity, + DisableQPSLimits: conf.DisableQPSLimits, ExtraArgs: kubeControllerManagerPatches.ExtraArgs, ExtraVolumes: kubeControllerManagerPatches.ExtraVolumes, }) @@ -368,6 +370,7 @@ func (c *Cluster) Install(ctx context.Context) error { KubeconfigPath: kubeconfigPath, KubeFeatureGates: conf.KubeFeatureGates, Verbosity: verbosity, + DisableQPSLimits: conf.DisableQPSLimits, ExtraArgs: kubeSchedulerComponentPatches.ExtraArgs, ExtraVolumes: kubeSchedulerComponentPatches.ExtraVolumes, }) diff --git a/pkg/kwokctl/runtime/compose/cluster.go b/pkg/kwokctl/runtime/compose/cluster.go index 41fc4daadb..d793c6ffdb 100644 --- a/pkg/kwokctl/runtime/compose/cluster.go +++ b/pkg/kwokctl/runtime/compose/cluster.go @@ -260,6 +260,7 @@ func (c *Cluster) Install(ctx context.Context) error { EtcdPort: conf.EtcdPort, EtcdAddress: c.Name() + "-etcd", Verbosity: verbosity, + DisableQPSLimits: conf.DisableQPSLimits, ExtraArgs: kubeApiserverComponentPatches.ExtraArgs, ExtraVolumes: kubeApiserverComponentPatches.ExtraVolumes, }) @@ -294,6 +295,7 @@ func (c *Cluster) Install(ctx context.Context) error { KubeconfigPath: inClusterOnHostKubeconfigPath, KubeFeatureGates: conf.KubeFeatureGates, Verbosity: verbosity, + DisableQPSLimits: conf.DisableQPSLimits, NodeMonitorPeriodMilliseconds: conf.KubeControllerManagerNodeMonitorPeriodMilliseconds, NodeMonitorGracePeriodMilliseconds: conf.KubeControllerManagerNodeMonitorGracePeriodMilliseconds, ExtraArgs: kubeControllerManagerComponentPatches.ExtraArgs, @@ -340,6 +342,7 @@ func (c *Cluster) Install(ctx context.Context) error { KubeconfigPath: inClusterOnHostKubeconfigPath, KubeFeatureGates: conf.KubeFeatureGates, Verbosity: verbosity, + DisableQPSLimits: conf.DisableQPSLimits, ExtraArgs: kubeSchedulerComponentPatches.ExtraArgs, ExtraVolumes: kubeSchedulerComponentPatches.ExtraVolumes, }) diff --git a/pkg/kwokctl/runtime/kind/cluster.go b/pkg/kwokctl/runtime/kind/cluster.go index b3ffa21d2e..64d4c3725e 100644 --- a/pkg/kwokctl/runtime/kind/cluster.go +++ b/pkg/kwokctl/runtime/kind/cluster.go @@ -41,6 +41,7 @@ import ( "sigs.k8s.io/kwok/pkg/utils/net" "sigs.k8s.io/kwok/pkg/utils/path" "sigs.k8s.io/kwok/pkg/utils/slices" + "sigs.k8s.io/kwok/pkg/utils/version" "sigs.k8s.io/kwok/pkg/utils/wait" ) @@ -132,6 +133,11 @@ func (c *Cluster) Install(ctx context.Context) error { configPath := c.GetWorkdirPath(runtime.ConfigName) + kubeVersion, err := version.ParseVersion(conf.KubeVersion) + if err != nil { + return err + } + etcdComponentPatches := runtime.GetComponentPatches(config, "etcd") kubeApiserverComponentPatches := runtime.GetComponentPatches(config, "kube-apiserver") kubeSchedulerComponentPatches := runtime.GetComponentPatches(config, "kube-scheduler") @@ -162,6 +168,8 @@ func (c *Cluster) Install(ctx context.Context) error { ControllerManagerExtraArgs: kubeControllerManagerComponentPatches.ExtraArgs, ControllerManagerExtraVolumes: kubeControllerManagerComponentPatches.ExtraVolumes, KwokControllerExtraVolumes: kwokControllerExtraVolumes, + DisableQPSLimits: conf.DisableQPSLimits, + KubeVersion: kubeVersion, }) if err != nil { return err diff --git a/pkg/kwokctl/runtime/kind/kind.go b/pkg/kwokctl/runtime/kind/kind.go index 7660864be5..fb8fbef0d0 100644 --- a/pkg/kwokctl/runtime/kind/kind.go +++ b/pkg/kwokctl/runtime/kind/kind.go @@ -19,12 +19,15 @@ package kind import ( "bytes" "fmt" + "strconv" "text/template" "sigs.k8s.io/kwok/pkg/apis/internalversion" + "sigs.k8s.io/kwok/pkg/consts" "sigs.k8s.io/kwok/pkg/kwokctl/runtime" "sigs.k8s.io/kwok/pkg/log" "sigs.k8s.io/kwok/pkg/utils/format" + "sigs.k8s.io/kwok/pkg/utils/version" _ "embed" ) @@ -137,6 +140,49 @@ func expendExtrasForBuildKind(conf BuildKindConfig) BuildKindConfig { }, ) } + + if conf.DisableQPSLimits { + conf.ApiserverExtraArgs = append(conf.ApiserverExtraArgs, + internalversion.ExtraArgs{ + Key: "max-requests-inflight", + Value: "0", + }, + internalversion.ExtraArgs{ + Key: "max-mutating-requests-inflight", + Value: "0", + }, + ) + + // FeatureGate APIPriorityAndFairness is not available before 1.17.0 + if conf.KubeVersion.GE(version.NewVersion(1, 18, 0)) { + conf.ApiserverExtraArgs = append(conf.ApiserverExtraArgs, + internalversion.ExtraArgs{ + Key: "enable-priority-and-fairness", + Value: "false", + }, + ) + } + conf.ControllerManagerExtraArgs = append(conf.ControllerManagerExtraArgs, + internalversion.ExtraArgs{ + Key: "kube-api-qps", + Value: strconv.FormatFloat(consts.DefaultUnlimitedQPS, 'f', 1, 64), + }, + internalversion.ExtraArgs{ + Key: "kube-api-burst", + Value: format.String(consts.DefaultUnlimitedBurst), + }, + ) + conf.SchedulerExtraArgs = append(conf.SchedulerExtraArgs, + internalversion.ExtraArgs{ + Key: "kube-api-qps", + Value: strconv.FormatFloat(consts.DefaultUnlimitedQPS, 'f', 1, 64), + }, + internalversion.ExtraArgs{ + Key: "kube-api-burst", + Value: format.String(consts.DefaultUnlimitedBurst), + }, + ) + } return conf } @@ -203,5 +249,7 @@ type BuildKindConfig struct { ControllerManagerExtraVolumes []internalversion.Volume KwokControllerExtraVolumes []internalversion.Volume - BindAddress string + BindAddress string + DisableQPSLimits bool + KubeVersion version.Version } diff --git a/site/content/en/docs/generated/apis.md b/site/content/en/docs/generated/apis.md index a2c2b96bc6..8a214ebbde 100644 --- a/site/content/en/docs/generated/apis.md +++ b/site/content/en/docs/generated/apis.md @@ -2428,6 +2428,17 @@ string

BindAddress is the address to bind to.

+ + +disableQPSLimits + +bool + + + +

DisableQPSLimits specifies whether to disable QPS limits for components.

+ +

diff --git a/site/content/en/docs/generated/kwokctl_create_cluster.md b/site/content/en/docs/generated/kwokctl_create_cluster.md index 6f791bdeb2..802a2bef05 100644 --- a/site/content/en/docs/generated/kwokctl_create_cluster.md +++ b/site/content/en/docs/generated/kwokctl_create_cluster.md @@ -12,6 +12,7 @@ kwokctl create cluster [flags] --controller-port uint32 Port of kwok-controller given to the host --disable-kube-controller-manager Disable the kube-controller-manager --disable-kube-scheduler Disable the kube-scheduler + --disable-qps-limits Disable QPS limits for components --docker-compose-binary string Binary of Docker-compose, only for docker runtime (default "https://github.com/docker/compose/releases/download/v2.17.2/docker-compose-linux-x86_64") --etcd-binary string Binary of etcd, only for binary runtime diff --git a/test/kwokctl/suite.sh b/test/kwokctl/suite.sh index e98d3a5c6d..5beb846fea 100644 --- a/test/kwokctl/suite.sh +++ b/test/kwokctl/suite.sh @@ -53,6 +53,7 @@ function create_cluster() { --timeout 30m \ --wait 30m \ --quiet-pull \ + --disable-qps-limits \ "$@"; then echo "Error: Cluster ${name} creation failed" show_all