Skip to content

Commit

Permalink
Merge pull request #691 from lianghao208/env-config
Browse files Browse the repository at this point in the history
add extra envs in KwokctlConfiguration
  • Loading branch information
k8s-ci-robot authored Jul 4, 2023
2 parents 94ba874 + f7e2cdc commit 71dc497
Show file tree
Hide file tree
Showing 28 changed files with 163 additions and 24 deletions.
2 changes: 2 additions & 0 deletions pkg/apis/config/v1alpha1/kwokctl_configuration_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ type ComponentPatches struct {
ExtraArgs []ExtraArgs `json:"extraArgs,omitempty"`
// ExtraVolumes is the extra volumes to be patched on the component.
ExtraVolumes []Volume `json:"extraVolumes,omitempty"`
// ExtraEnvs is the extra environment variables to be patched on the component.
ExtraEnvs []Env `json:"extraEnvs,omitempty"`
}

// KwokctlConfigurationOptions holds information about the options.
Expand Down
5 changes: 5 additions & 0 deletions pkg/apis/config/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 9 additions & 0 deletions pkg/apis/config/v1alpha1/zz_generated.defaults.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions pkg/apis/internalversion/kwokctl_configuration_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ type ComponentPatches struct {
ExtraArgs []ExtraArgs
// ExtraVolumes is the extra volumes to be patched on the component.
ExtraVolumes []Volume
// ExtraEnvs is the extra environment variables to be patched on the component.
ExtraEnvs []Env
}

// KwokctlConfigurationOptions holds information about the options.
Expand Down
2 changes: 2 additions & 0 deletions pkg/apis/internalversion/zz_generated.conversion.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions pkg/apis/internalversion/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions pkg/kwokctl/components/etcd.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ type BuildEtcdComponentConfig struct {
Verbosity log.Level
ExtraArgs []internalversion.ExtraArgs
ExtraVolumes []internalversion.Volume
ExtraEnvs []internalversion.Env
}

// BuildEtcdComponent builds an etcd component.
Expand Down Expand Up @@ -132,6 +133,7 @@ func BuildEtcdComponent(conf BuildEtcdComponentConfig) (component internalversio
Value: runtime.GOARCH,
})
}
envs = append(envs, conf.ExtraEnvs...)

return internalversion.Component{
Name: "etcd",
Expand Down
5 changes: 5 additions & 0 deletions pkg/kwokctl/components/kube_apiserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ type BuildKubeApiserverComponentConfig struct {
DisableQPSLimits bool
ExtraArgs []internalversion.ExtraArgs
ExtraVolumes []internalversion.Volume
ExtraEnvs []internalversion.Env
}

// BuildKubeApiserverComponent builds a kube-apiserver component.
Expand Down Expand Up @@ -218,6 +219,9 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon
kubeApiserverArgs = append(kubeApiserverArgs, "--v="+format.String(log.ToKlogLevel(conf.Verbosity)))
}

envs := []internalversion.Env{}
envs = append(envs, conf.ExtraEnvs...)

return internalversion.Component{
Name: "kube-apiserver",
Version: conf.Version.String(),
Expand All @@ -231,5 +235,6 @@ func BuildKubeApiserverComponent(conf BuildKubeApiserverComponentConfig) (compon
Binary: conf.Binary,
Image: conf.Image,
WorkDir: conf.Workdir,
Envs: envs,
}, nil
}
5 changes: 5 additions & 0 deletions pkg/kwokctl/components/kube_controller_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ type BuildKubeControllerManagerComponentConfig struct {
DisableQPSLimits bool
ExtraArgs []internalversion.ExtraArgs
ExtraVolumes []internalversion.Volume
ExtraEnvs []internalversion.Env
}

// BuildKubeControllerManagerComponent builds a kube-controller-manager component.
Expand Down Expand Up @@ -193,6 +194,9 @@ func BuildKubeControllerManagerComponent(conf BuildKubeControllerManagerComponen
kubeControllerManagerArgs = append(kubeControllerManagerArgs, "--v="+format.String(log.ToKlogLevel(conf.Verbosity)))
}

envs := []internalversion.Env{}
envs = append(envs, conf.ExtraEnvs...)

return internalversion.Component{
Name: "kube-controller-manager",
Version: conf.Version.String(),
Expand All @@ -206,5 +210,6 @@ func BuildKubeControllerManagerComponent(conf BuildKubeControllerManagerComponen
Binary: conf.Binary,
Image: conf.Image,
WorkDir: conf.Workdir,
Envs: envs,
}, nil
}
5 changes: 5 additions & 0 deletions pkg/kwokctl/components/kube_scheduler.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ type BuildKubeSchedulerComponentConfig struct {
DisableQPSLimits bool
ExtraArgs []internalversion.ExtraArgs
ExtraVolumes []internalversion.Volume
ExtraEnvs []internalversion.Env
}

// BuildKubeSchedulerComponent builds a kube-scheduler component.
Expand Down Expand Up @@ -183,6 +184,9 @@ func BuildKubeSchedulerComponent(conf BuildKubeSchedulerComponentConfig) (compon
kubeSchedulerArgs = append(kubeSchedulerArgs, "--v="+format.String(log.ToKlogLevel(conf.Verbosity)))
}

envs := []internalversion.Env{}
envs = append(envs, conf.ExtraEnvs...)

return internalversion.Component{
Name: "kube-scheduler",
Version: conf.Version.String(),
Expand All @@ -196,5 +200,6 @@ func BuildKubeSchedulerComponent(conf BuildKubeSchedulerComponentConfig) (compon
Image: conf.Image,
Ports: ports,
WorkDir: conf.Workdir,
Envs: envs,
}, nil
}
5 changes: 5 additions & 0 deletions pkg/kwokctl/components/kwok_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ type BuildKwokControllerComponentConfig struct {
NodeLeaseDurationSeconds uint
ExtraArgs []internalversion.ExtraArgs
ExtraVolumes []internalversion.Volume
ExtraEnvs []internalversion.Env
}

// BuildKwokControllerComponent builds a kwok controller component.
Expand Down Expand Up @@ -119,6 +120,9 @@ func BuildKwokControllerComponent(conf BuildKwokControllerComponentConfig) (comp
kwokControllerArgs = append(kwokControllerArgs, "--v="+format.String(conf.Verbosity))
}

envs := []internalversion.Env{}
envs = append(envs, conf.ExtraEnvs...)

return internalversion.Component{
Name: "kwok-controller",
Version: conf.Version.String(),
Expand All @@ -132,5 +136,6 @@ func BuildKwokControllerComponent(conf BuildKwokControllerComponentConfig) (comp
Binary: conf.Binary,
Image: conf.Image,
WorkDir: conf.Workdir,
Envs: envs,
}
}
5 changes: 5 additions & 0 deletions pkg/kwokctl/components/prometheus.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ type BuildPrometheusComponentConfig struct {
Verbosity log.Level
ExtraArgs []internalversion.ExtraArgs
ExtraVolumes []internalversion.Volume
ExtraEnvs []internalversion.Env
}

// BuildPrometheusComponent builds a prometheus component.
Expand Down Expand Up @@ -87,6 +88,9 @@ func BuildPrometheusComponent(conf BuildPrometheusComponentConfig) (component in
prometheusArgs = append(prometheusArgs, "--log.level="+log.ToLogSeverityLevel(conf.Verbosity))
}

envs := []internalversion.Env{}
envs = append(envs, conf.ExtraEnvs...)

return internalversion.Component{
Name: "prometheus",
Version: conf.Version.String(),
Expand All @@ -104,5 +108,6 @@ func BuildPrometheusComponent(conf BuildPrometheusComponentConfig) (component in
Binary: conf.Binary,
Image: conf.Image,
WorkDir: conf.Workdir,
Envs: envs,
}, nil
}
15 changes: 15 additions & 0 deletions pkg/kwokctl/runtime/binary/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,13 @@ import (
"sigs.k8s.io/kwok/pkg/kwokctl/dryrun"
"sigs.k8s.io/kwok/pkg/kwokctl/runtime"
"sigs.k8s.io/kwok/pkg/log"
"sigs.k8s.io/kwok/pkg/utils/exec"
"sigs.k8s.io/kwok/pkg/utils/file"
"sigs.k8s.io/kwok/pkg/utils/format"
"sigs.k8s.io/kwok/pkg/utils/kubeconfig"
"sigs.k8s.io/kwok/pkg/utils/net"
"sigs.k8s.io/kwok/pkg/utils/path"
"sigs.k8s.io/kwok/pkg/utils/slices"
"sigs.k8s.io/kwok/pkg/utils/wait"
)

Expand Down Expand Up @@ -281,6 +283,7 @@ func (c *Cluster) Install(ctx context.Context) error {
Verbosity: verbosity,
ExtraArgs: etcdComponentPatches.ExtraArgs,
ExtraVolumes: etcdComponentPatches.ExtraVolumes,
ExtraEnvs: etcdComponentPatches.ExtraEnvs,
})
if err != nil {
return err
Expand Down Expand Up @@ -316,6 +319,7 @@ func (c *Cluster) Install(ctx context.Context) error {
DisableQPSLimits: conf.DisableQPSLimits,
ExtraArgs: kubeApiserverComponentPatches.ExtraArgs,
ExtraVolumes: kubeApiserverComponentPatches.ExtraVolumes,
ExtraEnvs: kubeApiserverComponentPatches.ExtraEnvs,
})
if err != nil {
return err
Expand Down Expand Up @@ -356,6 +360,7 @@ func (c *Cluster) Install(ctx context.Context) error {
DisableQPSLimits: conf.DisableQPSLimits,
ExtraArgs: kubeControllerManagerPatches.ExtraArgs,
ExtraVolumes: kubeControllerManagerPatches.ExtraVolumes,
ExtraEnvs: kubeControllerManagerPatches.ExtraEnvs,
})
if err != nil {
return err
Expand Down Expand Up @@ -404,6 +409,7 @@ func (c *Cluster) Install(ctx context.Context) error {
DisableQPSLimits: conf.DisableQPSLimits,
ExtraArgs: kubeSchedulerComponentPatches.ExtraArgs,
ExtraVolumes: kubeSchedulerComponentPatches.ExtraVolumes,
ExtraEnvs: kubeSchedulerComponentPatches.ExtraEnvs,
})
if err != nil {
return err
Expand Down Expand Up @@ -434,6 +440,7 @@ func (c *Cluster) Install(ctx context.Context) error {
Verbosity: verbosity,
NodeLeaseDurationSeconds: conf.NodeLeaseDurationSeconds,
ExtraArgs: kwokControllerComponentPatches.ExtraArgs,
ExtraEnvs: kwokControllerComponentPatches.ExtraEnvs,
})
if err != nil {
return err
Expand Down Expand Up @@ -482,6 +489,7 @@ func (c *Cluster) Install(ctx context.Context) error {
Verbosity: verbosity,
ExtraArgs: prometheusComponentPatches.ExtraArgs,
ExtraVolumes: prometheusComponentPatches.ExtraVolumes,
ExtraEnvs: prometheusComponentPatches.ExtraEnvs,
})
if err != nil {
return err
Expand Down Expand Up @@ -539,6 +547,13 @@ func (c *Cluster) startComponent(ctx context.Context, component internalversion.
logger.Debug("Component already started")
return nil
}

if len(component.Envs) > 0 {
ctx = exec.WithEnv(ctx, slices.Map(component.Envs, func(c internalversion.Env) string {
return fmt.Sprintf("%s=%s", c.Name, c.Value)
}))
}

logger.Debug("Starting component")
return c.ForkExec(ctx, component.WorkDir, component.Binary, component.Args...)
}
Expand Down
6 changes: 6 additions & 0 deletions pkg/kwokctl/runtime/compose/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,7 @@ func (c *Cluster) Install(ctx context.Context) error {
Verbosity: verbosity,
ExtraArgs: etcdComponentPatches.ExtraArgs,
ExtraVolumes: etcdComponentPatches.ExtraVolumes,
ExtraEnvs: etcdComponentPatches.ExtraEnvs,
})
if err != nil {
return err
Expand Down Expand Up @@ -338,6 +339,7 @@ func (c *Cluster) Install(ctx context.Context) error {
DisableQPSLimits: conf.DisableQPSLimits,
ExtraArgs: kubeApiserverComponentPatches.ExtraArgs,
ExtraVolumes: kubeApiserverComponentPatches.ExtraVolumes,
ExtraEnvs: kubeApiserverComponentPatches.ExtraEnvs,
})
if err != nil {
return err
Expand Down Expand Up @@ -375,6 +377,7 @@ func (c *Cluster) Install(ctx context.Context) error {
NodeMonitorGracePeriodMilliseconds: conf.KubeControllerManagerNodeMonitorGracePeriodMilliseconds,
ExtraArgs: kubeControllerManagerComponentPatches.ExtraArgs,
ExtraVolumes: kubeControllerManagerComponentPatches.ExtraVolumes,
ExtraEnvs: kubeControllerManagerComponentPatches.ExtraEnvs,
})
if err != nil {
return err
Expand Down Expand Up @@ -420,6 +423,7 @@ func (c *Cluster) Install(ctx context.Context) error {
DisableQPSLimits: conf.DisableQPSLimits,
ExtraArgs: kubeSchedulerComponentPatches.ExtraArgs,
ExtraVolumes: kubeSchedulerComponentPatches.ExtraVolumes,
ExtraEnvs: kubeSchedulerComponentPatches.ExtraEnvs,
})
if err != nil {
return err
Expand Down Expand Up @@ -459,6 +463,7 @@ func (c *Cluster) Install(ctx context.Context) error {
NodeLeaseDurationSeconds: conf.NodeLeaseDurationSeconds,
ExtraArgs: kwokControllerComponentPatches.ExtraArgs,
ExtraVolumes: kwokControllerExtraVolumes,
ExtraEnvs: kwokControllerComponentPatches.ExtraEnvs,
})
config.Components = append(config.Components, kwokControllerComponent)

Expand Down Expand Up @@ -505,6 +510,7 @@ func (c *Cluster) Install(ctx context.Context) error {
Verbosity: verbosity,
ExtraArgs: prometheusComponentPatches.ExtraArgs,
ExtraVolumes: prometheusComponentPatches.ExtraVolumes,
ExtraEnvs: prometheusComponentPatches.ExtraEnvs,
})
if err != nil {
return err
Expand Down
9 changes: 9 additions & 0 deletions pkg/kwokctl/runtime/kind/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,12 @@ func (c *Cluster) Install(ctx context.Context) error {
extraLogVolumes := runtime.GetLogVolumes(ctx)
kwokControllerExtraVolumes := kwokControllerComponentPatches.ExtraVolumes
kwokControllerExtraVolumes = append(kwokControllerExtraVolumes, extraLogVolumes...)
if len(etcdComponentPatches.ExtraEnvs) > 0 ||
len(kubeApiserverComponentPatches.ExtraEnvs) > 0 ||
len(kubeSchedulerComponentPatches.ExtraEnvs) > 0 ||
len(kubeControllerManagerComponentPatches.ExtraEnvs) > 0 {
logger.Warn("extraEnvs config in etcd, kube-apiserver, kube-scheduler or kube-controller-manager is not supported in kind")
}
kindYaml, err := BuildKind(BuildKindConfig{
BindAddress: conf.BindAddress,
KubeApiserverPort: conf.KubeApiserverPort,
Expand All @@ -173,6 +179,7 @@ func (c *Cluster) Install(ctx context.Context) error {
ControllerManagerExtraArgs: kubeControllerManagerComponentPatches.ExtraArgs,
ControllerManagerExtraVolumes: kubeControllerManagerComponentPatches.ExtraVolumes,
KwokControllerExtraVolumes: kwokControllerExtraVolumes,
KwokControllerExtraExtraEnvs: kwokControllerComponentPatches.ExtraEnvs,
DisableQPSLimits: conf.DisableQPSLimits,
KubeVersion: kubeVersion,
})
Expand All @@ -191,6 +198,7 @@ func (c *Cluster) Install(ctx context.Context) error {
NodeLeaseDurationSeconds: 40,
ExtraArgs: kwokControllerComponentPatches.ExtraArgs,
ExtraVolumes: kwokControllerExtraVolumes,
ExtraEnvs: kwokControllerComponentPatches.ExtraEnvs,
})
if err != nil {
return err
Expand All @@ -207,6 +215,7 @@ func (c *Cluster) Install(ctx context.Context) error {
Name: c.Name(),
ExtraArgs: prometheusPatches.ExtraArgs,
ExtraVolumes: prometheusPatches.ExtraVolumes,
ExtraEnvs: prometheusPatches.ExtraEnvs,
Metrics: runtime.GetMetrics(ctx),
}
if verbosity != log.LevelInfo {
Expand Down
1 change: 1 addition & 0 deletions pkg/kwokctl/runtime/kind/kind.go
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,7 @@ type BuildKindConfig struct {
Verbosity log.Level
ControllerManagerExtraVolumes []internalversion.Volume
KwokControllerExtraVolumes []internalversion.Volume
KwokControllerExtraExtraEnvs []internalversion.Env

BindAddress string
DisableQPSLimits bool
Expand Down
1 change: 1 addition & 0 deletions pkg/kwokctl/runtime/kind/kwok_controller_pod.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,4 +64,5 @@ type BuildKwokControllerPodConfig struct {
NodeLeaseDurationSeconds uint
ExtraArgs []internalversion.ExtraArgs
ExtraVolumes []internalversion.Volume
ExtraEnvs []internalversion.Env
}
4 changes: 4 additions & 0 deletions pkg/kwokctl/runtime/kind/kwok_controller_pod.yaml.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@ spec:
valueFrom:
fieldRef:
fieldPath: status.podIP
{{ range .ExtraEnvs }}
- name: {{ .Name }}
value: {{ .Value }}
{{ end }}
image: '{{.KwokControllerImageName}}:{{.KwokControllerImageTag}}'
imagePullPolicy: IfNotPresent
livenessProbe:
Expand Down
Loading

0 comments on commit 71dc497

Please sign in to comment.