Skip to content

Commit

Permalink
kwokctl: add --heartbeat-factor for cluster creation parameters
Browse files Browse the repository at this point in the history
  • Loading branch information
wzshiming committed Jan 31, 2024
1 parent d9ae3b6 commit 140fd56
Show file tree
Hide file tree
Showing 21 changed files with 71 additions and 19 deletions.
10 changes: 7 additions & 3 deletions pkg/apis/config/v1alpha1/kwokctl_configuration_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -425,21 +425,25 @@ type KwokctlConfigurationOptions struct {
CacheDir string `json:"cacheDir,omitempty"`

// KubeControllerManagerNodeMonitorPeriodMilliseconds is --node-monitor-period for kube-controller-manager.
// +default=600000
// +default=5000
KubeControllerManagerNodeMonitorPeriodMilliseconds int64 `json:"kubeControllerManagerNodeMonitorPeriodMilliseconds,omitempty"`

// KubeControllerManagerNodeMonitorGracePeriodMilliseconds is --node-monitor-grace-period for kube-controller-manager.
// +default=3600000
// +default=40000
KubeControllerManagerNodeMonitorGracePeriodMilliseconds int64 `json:"kubeControllerManagerNodeMonitorGracePeriodMilliseconds,omitempty"`

// NodeStatusUpdateFrequencyMilliseconds is --node-status-update-frequency for kwok like kubelet.
// +default=1200000
// +default=10000
NodeStatusUpdateFrequencyMilliseconds int64 `json:"nodeStatusUpdateFrequencyMilliseconds,omitempty"`

// NodeLeaseDurationSeconds is the duration the Kubelet will set on its corresponding Lease.
// +default=40
NodeLeaseDurationSeconds uint `json:"nodeLeaseDurationSeconds,omitempty"`

// HeartbeatFactor is the scale factor for all about heartbeat.
// +default=5
HeartbeatFactor *float64 `json:"heartbeatFactor,omitempty"`

// BindAddress is the address to bind to.
// +default="0.0.0.0"
BindAddress string `json:"bindAddress,omitempty"`
Expand Down
5 changes: 5 additions & 0 deletions pkg/apis/config/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 7 additions & 3 deletions pkg/apis/config/v1alpha1/zz_generated.defaults.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions pkg/apis/internalversion/kwokctl_configuration_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,9 @@ type KwokctlConfigurationOptions struct {
// NodeLeaseDurationSeconds is the duration the Kubelet will set on its corresponding Lease.
NodeLeaseDurationSeconds uint

// HeartbeatFactor is the scale factor for all about heartbeat.
HeartbeatFactor float64

// BindAddress is the address to bind to.
BindAddress string

Expand Down
6 changes: 6 additions & 0 deletions pkg/apis/internalversion/zz_generated.conversion.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

18 changes: 18 additions & 0 deletions pkg/kwokctl/cmd/create/cluster/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -144,10 +144,26 @@ func NewCommand(ctx context.Context) *cobra.Command {
cmd.Flags().BoolVar(&flags.Options.DisableQPSLimits, "disable-qps-limits", flags.Options.DisableQPSLimits, "Disable QPS limits for components")
cmd.Flags().StringSliceVar(&flags.Options.EnableCRDs, "enable-crds", flags.Options.EnableCRDs, "List of CRDs to enable")
cmd.Flags().UintVar(&flags.Options.NodeLeaseDurationSeconds, "node-lease-duration-seconds", flags.Options.NodeLeaseDurationSeconds, "Duration of node lease in seconds")
cmd.Flags().Float64Var(&flags.Options.HeartbeatFactor, "heartbeat-factor", flags.Options.HeartbeatFactor, "Scale factor for all about heartbeat")

return cmd
}

func multiplyByFactor[T ~int64 | ~uint](num *T, factor float64) {
*num = T(float64(*num) * factor)
}

func mutationHeartbeat(flags *flagpole) {
if flags.Options.HeartbeatFactor == 0 || flags.Options.HeartbeatFactor == 1 {
return
}
multiplyByFactor(&flags.Options.NodeLeaseDurationSeconds, flags.Options.HeartbeatFactor)
multiplyByFactor(&flags.Options.NodeStatusUpdateFrequencyMilliseconds, flags.Options.HeartbeatFactor)
multiplyByFactor(&flags.Options.KubeControllerManagerNodeMonitorGracePeriodMilliseconds, flags.Options.HeartbeatFactor)
multiplyByFactor(&flags.Options.KubeControllerManagerNodeMonitorPeriodMilliseconds, flags.Options.HeartbeatFactor)
flags.Options.HeartbeatFactor = 1
}

func runE(ctx context.Context, flags *flagpole) error {
name := config.ClusterName(flags.Name)
workdir := path.Join(config.ClustersDir, flags.Name)
Expand All @@ -171,6 +187,8 @@ func runE(ctx context.Context, flags *flagpole) error {
defer cancel()
}

mutationHeartbeat(flags)

// Choose runtime
var rt runtime.Runtime
if flags.Options.Runtime == "" {
Expand Down
11 changes: 11 additions & 0 deletions site/content/en/docs/generated/apis.md
Original file line number Diff line number Diff line change
Expand Up @@ -3634,6 +3634,17 @@ uint
</tr>
<tr>
<td>
<code>heartbeatFactor</code>
<em>
float64
</em>
</td>
<td>
<p>HeartbeatFactor is the scale factor for all about heartbeat.</p>
</td>
</tr>
<tr>
<td>
<code>bindAddress</code>
<em>
string
Expand Down
1 change: 1 addition & 0 deletions site/content/en/docs/generated/kwokctl_create_cluster.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ kwokctl create cluster [flags]
(default "registry.k8s.io/etcd:3.5.11-0")
--etcd-port uint32 Port of etcd given to the host. The behavior is unstable for kind/kind-podman runtime and may be modified in the future
--etcd-prefix string prefix of the key (default "/registry")
--heartbeat-factor float Scale factor for all about heartbeat (default 5)
-h, --help help for cluster
--jaeger-binary string Binary of Jaeger, only for binary runtime (default "https://github.com/jaegertracing/jaeger/releases/download/v1.53.0/jaeger-1.53.0-linux-amd64.tar.gz#jaeger-all-in-one")
--jaeger-image string Image of Jaeger, only for docker/podman/nerdctl/kind/kind-podman runtime
Expand Down
2 changes: 1 addition & 1 deletion test/kwokctl/testdata/binary/create_cluster.txt
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ cd <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME> && etcd --name=node0 --auto-compac
echo $! ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pids/etcd.pid
cd <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME> && kube-apiserver --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://127.0.0.1:32765 --bind-address=0.0.0.0 --secure-port=32764 --tls-cert-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt --tls-private-key-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key --client-ca-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/ca.crt --service-account-key-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key --service-account-signing-key-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key --proxy-client-cert-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/logs/kube-apiserver.log 2>&1 &
echo $! ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pids/kube-apiserver.pid
cd <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME> && kube-controller-manager --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32762 --kube-api-qps=5000 --kube-api-burst=10000 ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/logs/kube-controller-manager.log 2>&1 &
cd <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME> && kube-controller-manager --node-monitor-period=5s --node-monitor-grace-period=40s --kubeconfig=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32762 --kube-api-qps=5000 --kube-api-burst=10000 ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/logs/kube-controller-manager.log 2>&1 &
echo $! ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pids/kube-controller-manager.pid
cd <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME> && kube-scheduler --kubeconfig=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32761 --kube-api-qps=5000 --kube-api-burst=10000 ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/logs/kube-scheduler.log 2>&1 &
echo $! ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pids/kube-scheduler.pid
Expand Down
2 changes: 1 addition & 1 deletion test/kwokctl/testdata/binary/create_cluster_with_extra.txt
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ cd <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME> && TEST_KEY=TEST_VALUE etcd --name
echo $! ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pids/etcd.pid
cd <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME> && TEST_KEY=TEST_VALUE kube-apiserver --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://127.0.0.1:32765 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=32764 --tls-cert-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt --tls-private-key-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key --client-ca-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/ca.crt --service-account-key-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key --service-account-signing-key-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key --proxy-client-cert-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt --v=5 ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/logs/kube-apiserver.log 2>&1 &
echo $! ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pids/kube-apiserver.pid
cd <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME> && TEST_KEY=TEST_VALUE kube-controller-manager --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32762 --root-ca-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/ca.crt --service-account-private-key-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=5 ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/logs/kube-controller-manager.log 2>&1 &
cd <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME> && TEST_KEY=TEST_VALUE kube-controller-manager --node-monitor-period=5s --node-monitor-grace-period=40s --kubeconfig=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32762 --root-ca-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/ca.crt --service-account-private-key-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=5 ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/logs/kube-controller-manager.log 2>&1 &
echo $! ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pids/kube-controller-manager.pid
cd <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME> && TEST_KEY=TEST_VALUE kube-scheduler --kubeconfig=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32761 --kube-api-qps=5000 --kube-api-burst=10000 --v=5 ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/logs/kube-scheduler.log 2>&1 &
echo $! ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pids/kube-scheduler.pid
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ cd <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME> && jaeger-all-in-one --collector.o
echo $! ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pids/jaeger-all-in-one.pid
cd <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME> && kube-apiserver --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://127.0.0.1:32765 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=32764 --tls-cert-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt --tls-private-key-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key --client-ca-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/ca.crt --service-account-key-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key --service-account-signing-key-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key --proxy-client-cert-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt --audit-policy-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/audit.yaml --audit-log-path=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/logs/audit.log --tracing-config-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/apiserver-tracing-config.yaml --v=4 ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/logs/kube-apiserver.log 2>&1 &
echo $! ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pids/kube-apiserver.pid
cd <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME> && kube-controller-manager --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32761 --root-ca-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/ca.crt --service-account-private-key-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=4 ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/logs/kube-controller-manager.log 2>&1 &
cd <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME> && kube-controller-manager --node-monitor-period=5s --node-monitor-grace-period=40s --kubeconfig=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32761 --root-ca-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/ca.crt --service-account-private-key-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 --v=4 ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/logs/kube-controller-manager.log 2>&1 &
echo $! ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pids/kube-controller-manager.pid
cd <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME> && kube-scheduler --config=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/scheduler.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32760 --kube-api-qps=5000 --kube-api-burst=10000 --v=4 ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/logs/kube-scheduler.log 2>&1 &
echo $! ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pids/kube-scheduler.pid
Expand Down
2 changes: 1 addition & 1 deletion test/kwokctl/testdata/binary/start_cluster.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ cd <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME> && etcd --name=node0 --auto-compac
echo $! ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pids/etcd.pid
cd <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME> && kube-apiserver --etcd-prefix=/registry --allow-privileged=true --max-requests-inflight=0 --max-mutating-requests-inflight=0 --enable-priority-and-fairness=false --etcd-servers=http://127.0.0.1:32765 --authorization-mode=Node,RBAC --bind-address=0.0.0.0 --secure-port=32764 --tls-cert-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt --tls-private-key-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key --client-ca-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/ca.crt --service-account-key-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key --service-account-signing-key-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key --service-account-issuer=https://kubernetes.default.svc.cluster.local --proxy-client-key-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key --proxy-client-cert-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/logs/kube-apiserver.log 2>&1 &
echo $! ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pids/kube-apiserver.pid
cd <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME> && kube-controller-manager --node-monitor-period=10m0s --node-monitor-grace-period=1h0m0s --kubeconfig=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32762 --root-ca-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/ca.crt --service-account-private-key-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/logs/kube-controller-manager.log 2>&1 &
cd <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME> && kube-controller-manager --node-monitor-period=5s --node-monitor-grace-period=40s --kubeconfig=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32762 --root-ca-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/ca.crt --service-account-private-key-file=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key --kube-api-qps=5000 --kube-api-burst=10000 ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/logs/kube-controller-manager.log 2>&1 &
echo $! ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pids/kube-controller-manager.pid
cd <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME> && kube-scheduler --kubeconfig=<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/kubeconfig.yaml --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics --bind-address=0.0.0.0 --secure-port=32761 --kube-api-qps=5000 --kube-api-burst=10000 ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/logs/kube-scheduler.log 2>&1 &
echo $! ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pids/kube-scheduler.pid
Expand Down
Loading

0 comments on commit 140fd56

Please sign in to comment.