diff --git a/README.md b/README.md index 379caf88f..50d3bac54 100644 --- a/README.md +++ b/README.md @@ -41,15 +41,15 @@ We have several components: ### Simulator -Simulator is kube-apiserver + scheduler + the HTTP server which mainly for the web UI. +Simulator is kube-apiserver with some controllers + debuggable scheduler + the HTTP server which mainly for the web UI. There are several ways to integrate your scheduler into the simulator. See [integrate-your-scheduler.md](simulator/docs/integrate-your-scheduler.md). -You can create any resources by communicating with kube-apiserver in any ways (kubectl, k8s client library, or web UI described next) -and see how your Pods are scheduled. - -When you create Pods, Pods will get annotations from the simulator which contains the scheduling results per plugins or extenders. +You can create any resources by any ways (kubectl, k8s client library, or web UI described next). +And when you create Pods, +Pods will be scheduled by the [debuggable scheduler](./simulator/docs/debuggable-scheduler.md), +and they'll get the annotations that explain how each Pod was evaluated by each scheduler plugin. ```yaml kind: Pod @@ -81,7 +81,8 @@ metadata: You can utilize these results to understand your scheduler, check/test your configurations or customized scheduler's behavior. -Further expansion, you can export internal state more, change specific behaviours on plugins etc by implementing [PluginExtender](./simulator/docs/plugin-extender.md). +Further expansion, you can export internal state more, or change specific behaviours on plugins +by implementing [PluginExtender](./simulator/docs/plugin-extender.md). The simulator has its own configuration, you can refer to the [documentation](./simulator/docs/simulator-server-config.md). diff --git a/simulator/docs/debuggable-scheduler.md b/simulator/docs/debuggable-scheduler.md new file mode 100644 index 000000000..ce2ed4ad5 --- /dev/null +++ b/simulator/docs/debuggable-scheduler.md @@ -0,0 +1,94 @@ +## Debuggable scheduler + +The "debuggable scheduler" is the core concept of the simulator. + +### What's the "debuggable scheduler"? + +The debuggable scheduler is the scheduler that schedules Pods like other schedulers, +but it exposes the scheduling results from each scheduling plugin on Pod annotation. + +```yaml +kind: Pod +apiVersion: v1 +metadata: + name: hoge-pod + annotations: + scheduler-simulator/bind-result: '{"DefaultBinder":"success"}' + scheduler-simulator/filter-result: >- + {"node-282x7":{"AzureDiskLimits":"passed","EBSLimits":"passed","GCEPDLimits":"passed","InterPodAffinity":"passed","NodeAffinity":"passed","NodeName":"passed","NodePorts":"passed","NodeResourcesFit":"passed","NodeUnschedulable":"passed","NodeVolumeLimits":"passed","PodTopologySpread":"passed","TaintToleration":"passed","VolumeBinding":"passed","VolumeRestrictions":"passed","VolumeZone":"passed"},"node-gp9t4":{"AzureDiskLimits":"passed","EBSLimits":"passed","GCEPDLimits":"passed","InterPodAffinity":"passed","NodeAffinity":"passed","NodeName":"passed","NodePorts":"passed","NodeResourcesFit":"passed","NodeUnschedulable":"passed","NodeVolumeLimits":"passed","PodTopologySpread":"passed","TaintToleration":"passed","VolumeBinding":"passed","VolumeRestrictions":"passed","VolumeZone":"passed"}} + scheduler-simulator/finalscore-result: >- + {"node-282x7":{"ImageLocality":"0","InterPodAffinity":"0","NodeAffinity":"0","NodeNumber":"0","NodeResourcesBalancedAllocation":"76","NodeResourcesFit":"73","PodTopologySpread":"200","TaintToleration":"300","VolumeBinding":"0"},"node-gp9t4":{"ImageLocality":"0","InterPodAffinity":"0","NodeAffinity":"0","NodeNumber":"0","NodeResourcesBalancedAllocation":"76","NodeResourcesFit":"73","PodTopologySpread":"200","TaintToleration":"300","VolumeBinding":"0"}} + scheduler-simulator/permit-result: '{}' + scheduler-simulator/permit-result-timeout: '{}' + scheduler-simulator/postfilter-result: '{}' + scheduler-simulator/prebind-result: '{"VolumeBinding":"success"}' + scheduler-simulator/prefilter-result: '{}' + scheduler-simulator/prefilter-result-status: >- + {"InterPodAffinity":"success","NodeAffinity":"success","NodePorts":"success","NodeResourcesFit":"success","PodTopologySpread":"success","VolumeBinding":"success","VolumeRestrictions":"success"} + scheduler-simulator/prescore-result: >- + {"InterPodAffinity":"success","NodeAffinity":"success","NodeNumber":"success","PodTopologySpread":"success","TaintToleration":"success"} + scheduler-simulator/reserve-result: '{"VolumeBinding":"success"}' + scheduler-simulator/result-history: >- + [{"noderesourcefit-prefilter-data":"{\"MilliCPU\":100,\"Memory\":17179869184,\"EphemeralStorage\":0,\"AllowedPodNumber\":0,\"ScalarResources\":null}","scheduler-simulator/bind-result":"{\"DefaultBinder\":\"success\"}","scheduler-simulator/filter-result":"{\"node-282x7\":{\"AzureDiskLimits\":\"passed\",\"EBSLimits\":\"passed\",\"GCEPDLimits\":\"passed\",\"InterPodAffinity\":\"passed\",\"NodeAffinity\":\"passed\",\"NodeName\":\"passed\",\"NodePorts\":\"passed\",\"NodeResourcesFit\":\"passed\",\"NodeUnschedulable\":\"passed\",\"NodeVolumeLimits\":\"passed\",\"PodTopologySpread\":\"passed\",\"TaintToleration\":\"passed\",\"VolumeBinding\":\"passed\",\"VolumeRestrictions\":\"passed\",\"VolumeZone\":\"passed\"},\"node-gp9t4\":{\"AzureDiskLimits\":\"passed\",\"EBSLimits\":\"passed\",\"GCEPDLimits\":\"passed\",\"InterPodAffinity\":\"passed\",\"NodeAffinity\":\"passed\",\"NodeName\":\"passed\",\"NodePorts\":\"passed\",\"NodeResourcesFit\":\"passed\",\"NodeUnschedulable\":\"passed\",\"NodeVolumeLimits\":\"passed\",\"PodTopologySpread\":\"passed\",\"TaintToleration\":\"passed\",\"VolumeBinding\":\"passed\",\"VolumeRestrictions\":\"passed\",\"VolumeZone\":\"passed\"}}","scheduler-simulator/finalscore-result":"{\"node-282x7\":{\"ImageLocality\":\"0\",\"InterPodAffinity\":\"0\",\"NodeAffinity\":\"0\",\"NodeNumber\":\"0\",\"NodeResourcesBalancedAllocation\":\"76\",\"NodeResourcesFit\":\"73\",\"PodTopologySpread\":\"200\",\"TaintToleration\":\"300\",\"VolumeBinding\":\"0\"},\"node-gp9t4\":{\"ImageLocality\":\"0\",\"InterPodAffinity\":\"0\",\"NodeAffinity\":\"0\",\"NodeNumber\":\"0\",\"NodeResourcesBalancedAllocation\":\"76\",\"NodeResourcesFit\":\"73\",\"PodTopologySpread\":\"200\",\"TaintToleration\":\"300\",\"VolumeBinding\":\"0\"}}","scheduler-simulator/permit-result":"{}","scheduler-simulator/permit-result-timeout":"{}","scheduler-simulator/postfilter-result":"{}","scheduler-simulator/prebind-result":"{\"VolumeBinding\":\"success\"}","scheduler-simulator/prefilter-result":"{}","scheduler-simulator/prefilter-result-status":"{\"InterPodAffinity\":\"success\",\"NodeAffinity\":\"success\",\"NodePorts\":\"success\",\"NodeResourcesFit\":\"success\",\"PodTopologySpread\":\"success\",\"VolumeBinding\":\"success\",\"VolumeRestrictions\":\"success\"}","scheduler-simulator/prescore-result":"{\"InterPodAffinity\":\"success\",\"NodeAffinity\":\"success\",\"NodeNumber\":\"success\",\"PodTopologySpread\":\"success\",\"TaintToleration\":\"success\"}","scheduler-simulator/reserve-result":"{\"VolumeBinding\":\"success\"}","scheduler-simulator/score-result":"{\"node-282x7\":{\"ImageLocality\":\"0\",\"InterPodAffinity\":\"0\",\"NodeAffinity\":\"0\",\"NodeNumber\":\"0\",\"NodeResourcesBalancedAllocation\":\"76\",\"NodeResourcesFit\":\"73\",\"PodTopologySpread\":\"0\",\"TaintToleration\":\"0\",\"VolumeBinding\":\"0\"},\"node-gp9t4\":{\"ImageLocality\":\"0\",\"InterPodAffinity\":\"0\",\"NodeAffinity\":\"0\",\"NodeNumber\":\"0\",\"NodeResourcesBalancedAllocation\":\"76\",\"NodeResourcesFit\":\"73\",\"PodTopologySpread\":\"0\",\"TaintToleration\":\"0\",\"VolumeBinding\":\"0\"}}","scheduler-simulator/selected-node":"node-282x7"}] + scheduler-simulator/score-result: >- + {"node-282x7":{"ImageLocality":"0","InterPodAffinity":"0","NodeAffinity":"0","NodeNumber":"0","NodeResourcesBalancedAllocation":"76","NodeResourcesFit":"73","PodTopologySpread":"0","TaintToleration":"0","VolumeBinding":"0"},"node-gp9t4":{"ImageLocality":"0","InterPodAffinity":"0","NodeAffinity":"0","NodeNumber":"0","NodeResourcesBalancedAllocation":"76","NodeResourcesFit":"73","PodTopologySpread":"0","TaintToleration":"0","VolumeBinding":"0"}} + scheduler-simulator/selected-node: node-282x7 +``` + +The simulator runs the debuggable scheduler built from the upstream scheduler by default, +that's why we can see the scheduling results like the above in the simulator. + +## Build a debuggable scheduler from your scheduler + +You can use the [`debuggablescheduler` package](../pkg/debuggablescheduler) to transform your scheduler into a debuggable scheduler. + +The debuggable scheduler could work anywhere, in the simulator or even in your cluster. + +### Change your scheduler + +Here, we assume you're registering your custom plugins in your scheduler like this: + +```go +// your scheduler's main package +func main() { + command := app.NewSchedulerCommand( + app.WithPlugin(yourcustomplugin.Name, yourcustomplugin.New), + ) + + code := cli.Run(command) + os.Exit(code) +} +``` + +Then, you need to replace few lines to use the [`debuggablescheduler` package](../pkg/debuggablescheduler). + +```go +func main() { + command, cancelFn, err := debuggablescheduler.NewSchedulerCommand( + debuggablescheduler.WithPlugin(yourcustomplugin.Name, yourcustomplugin.New), + debuggablescheduler.WithPluginExtenders(noderesources.Name, extender.New), // [optional] see plugin-extender.md about PluginExtender. + ) + if err != nil { + klog.Info(fmt.Sprintf("failed to build the scheduler command: %+v", err)) + os.Exit(1) + } + code := cli.Run(command) + cancelFn() + os.Exit(code) +} +``` + +As you see, `debuggablescheduler.NewSchedulerCommand` has much similar interface as the `app.NewSchedulerCommand`. +You can register your plugins by `debuggablescheduler.WithPlugin` option. + +### The plugin extender + +We have the plugin extender feature to expand the debuggable scheduler more. +See [plugin-extender.md](./plugin-extender.md) to know more about it. + +### The example debuggable scheduler + +We have the sample implementation in [./sample/debuggable-scheduler](./sample/debuggable-scheduler). + +You can try to run it in the simulator via [external scheduler](./external-scheduler.md) feature. +See [external-scheduler.md#the-example-external-scheduler](./external-scheduler.md#the-example-external-scheduler). diff --git a/simulator/docs/external-scheduler.md b/simulator/docs/external-scheduler.md index 7a31fd536..a74096807 100644 --- a/simulator/docs/external-scheduler.md +++ b/simulator/docs/external-scheduler.md @@ -1,67 +1,16 @@ ## External scheduler -This document describes about the feature called "external scheduler" +The external scheduler is the feature of the simulator +to allow you to run any of your scheduler in the simulator instead of the default one. -We use the [`externalscheduler` package](../pkg/externalscheduler); -the scheduler built with the [`externalscheduler` package](../pkg/externalscheduler) will export the scheduling results on each Pod annotation. +Before diving into the description, you need to know [debuggable scheduler](./debuggable-scheduler.md). -### Use cases +### connect your scheduler to the simulator -- Running your scheduler instead of the default one in the simulator - - You can still see the scheduling results in web UI as well! -- Running your scheduler with the simulator's feature in your clusterr. - - Like in the simulator, all Pods will get the scheduling results on its annotation while each scheduling is done as usual. - - Note that it has performance overhead in each scheduling cycle. -since the scheduler needs to make additional effort to export the scheduling results. - -### Change your scheduler - -Here, we assume you're registering your custom plugins in your scheduler like this: - -```go -// your scheduler's main package -func main() { - command := app.NewSchedulerCommand( - app.WithPlugin(yourcustomplugin.Name, yourcustomplugin.New), - ) - - code := cli.Run(command) - os.Exit(code) -} -``` - -Then, you need to replace few lines to use the [`externalscheduler` package](../pkg/externalscheduler). - -```go -func main() { - command, cancelFn, err := externalscheduler.NewSchedulerCommand( - externalscheduler.WithPlugin(yourcustomplugin.Name, yourcustomplugin.New), - externalscheduler.WithPluginExtenders(noderesources.Name, extender.New), // [optional] see plugin-extender.md about PluginExtender. - ) - if err != nil { - klog.Info(fmt.Sprintf("failed to build the scheduler command: %+v", err)) - os.Exit(1) - } - code := cli.Run(command) - cancelFn() - os.Exit(code) -} -``` - -As you see, `externalscheduler.NewSchedulerCommand` has much similar interface as the `app.NewSchedulerCommand`. -You can register your plugins by `externalscheduler.WithPlugin` option. - -Via this step, all Pods scheduled by this scheduler will get the scheduling results in the annotation like in the simulator! - -### Connect your scheduler to the kube-apiserver in the simulator - -If you are here to run the scheduler built with [`externalscheduler` package](../pkg/externalscheduler) in your cluster, -you don't need to follow this step. - -Let's connect your scheduler into the simulator. +Let's connect your scheduler to the simulator. First, you need to set `externalSchedulerEnabled: true` on [the simulator config](../config.yaml) -so that the scheduler in the simulator won't get started. +so that the scheduler, running in the simulator by default, won't get started. Next, you need to connect your scheduler into the simulator's kube-apiserver via KubeSchedulerConfig: @@ -76,14 +25,15 @@ You can use this [kubeconfig.yaml](./kubeconfig.yaml) to communicate with the si ### The example external scheduler -We have the sample external scheduler implementation in [./sample/external-scheduler](./sample/external-scheduler). +You can see how the external scheduler can be set up +with the sample debuggable scheduler implementation in [./sample/debuggable-scheduler](./sample/debuggable-scheduler). prerequisite: 1. set `externalSchedulerEnabled: true` on [the simulator config](../config.yaml) 2. run the simulator ```shell -cd sample/external-scheduler +cd sample/debuggable-scheduler go run main.go --config scheduler.yaml ``` diff --git a/simulator/docs/plugin-extender.md b/simulator/docs/plugin-extender.md index 1a01f00c8..706ddf3cf 100644 --- a/simulator/docs/plugin-extender.md +++ b/simulator/docs/plugin-extender.md @@ -50,13 +50,13 @@ each Pod will get `"noderesourcefit-prefilter-data": prefilterData` annotation i **Currently, the plugin extender can be used only in [the external scheduler](./external-scheduler.md).** -You can use `externalscheduler.WithPluginExtenders` option in `externalscheduler.NewSchedulerCommand` +You can use `debuggablescheduler.WithPluginExtenders` option in `debuggablescheduler.NewSchedulerCommand` to enable some PluginExtender in particular plugin. ```go func main() { - command, cancelFn, err := externalscheduler.NewSchedulerCommand( - externalscheduler.WithPluginExtenders(noderesources.Name, extender.New), + command, cancelFn, err := debuggablescheduler.NewSchedulerCommand( + debuggablescheduler.WithPluginExtenders(noderesources.Name, extender.New), ) if err != nil { klog.Info(fmt.Sprintf("failed to build the scheduler command: %+v", err)) diff --git a/simulator/docs/sample/external-scheduler/kubeconfig.yaml b/simulator/docs/sample/debuggable-scheduler/kubeconfig.yaml similarity index 100% rename from simulator/docs/sample/external-scheduler/kubeconfig.yaml rename to simulator/docs/sample/debuggable-scheduler/kubeconfig.yaml diff --git a/simulator/docs/sample/external-scheduler/main.go b/simulator/docs/sample/debuggable-scheduler/main.go similarity index 64% rename from simulator/docs/sample/external-scheduler/main.go rename to simulator/docs/sample/debuggable-scheduler/main.go index df8dad541..c48831dd4 100644 --- a/simulator/docs/sample/external-scheduler/main.go +++ b/simulator/docs/sample/debuggable-scheduler/main.go @@ -14,13 +14,13 @@ import ( "sigs.k8s.io/kube-scheduler-simulator/simulator/docs/sample/nodenumber" extender "sigs.k8s.io/kube-scheduler-simulator/simulator/docs/sample/plugin-extender" - "sigs.k8s.io/kube-scheduler-simulator/simulator/pkg/externalscheduler" + "sigs.k8s.io/kube-scheduler-simulator/simulator/pkg/debuggablescheduler" ) func main() { - command, cancelFn, err := externalscheduler.NewSchedulerCommand( - externalscheduler.WithPlugin(nodenumber.Name, nodenumber.New), - externalscheduler.WithPluginExtenders(noderesources.Name, extender.New), + command, cancelFn, err := debuggablescheduler.NewSchedulerCommand( + debuggablescheduler.WithPlugin(nodenumber.Name, nodenumber.New), // Register the custom scheduler plugin. + debuggablescheduler.WithPluginExtenders(noderesources.Name, extender.New), // [Optional] Register the plugin extender. See /simulator/docs/plugin-extender.md ) if err != nil { klog.Info(fmt.Sprintf("failed to build the scheduler command: %+v", err)) diff --git a/simulator/docs/sample/external-scheduler/scheduler.yaml b/simulator/docs/sample/debuggable-scheduler/scheduler.yaml similarity index 100% rename from simulator/docs/sample/external-scheduler/scheduler.yaml rename to simulator/docs/sample/debuggable-scheduler/scheduler.yaml diff --git a/simulator/pkg/debuggablescheduler/command.go b/simulator/pkg/debuggablescheduler/command.go new file mode 100644 index 000000000..902848224 --- /dev/null +++ b/simulator/pkg/debuggablescheduler/command.go @@ -0,0 +1,46 @@ +package debuggablescheduler + +import ( + "github.com/spf13/cobra" + "k8s.io/kubernetes/cmd/kube-scheduler/app" + "k8s.io/kubernetes/pkg/scheduler/framework/runtime" + + "sigs.k8s.io/kube-scheduler-simulator/simulator/scheduler/plugin" +) + +func NewSchedulerCommand(opts ...Option) (*cobra.Command, func(), error) { + opt := &options{pluginExtender: map[string]plugin.PluginExtenderInitializer{}, outOfTreeRegistry: map[string]runtime.PluginFactory{}} + for _, o := range opts { + o(opt) + } + + scheduleropts, cancelFn, err := CreateOptionForOutOfTreePlugin(opt.outOfTreeRegistry, opt.pluginExtender) + if err != nil { + return nil, cancelFn, err + } + + command := app.NewSchedulerCommand(scheduleropts...) + + return command, cancelFn, nil +} + +type options struct { + outOfTreeRegistry runtime.Registry + pluginExtender map[string]plugin.PluginExtenderInitializer +} + +type Option func(opt *options) + +// WithPlugin creates an Option based on plugin name and factory. +func WithPlugin(pluginName string, factory runtime.PluginFactory) Option { + return func(opt *options) { + opt.outOfTreeRegistry[pluginName] = factory + } +} + +// WithPluginExtenders creates an Option based on plugin name and plugin extenders. +func WithPluginExtenders(pluginName string, e plugin.PluginExtenderInitializer) Option { + return func(opt *options) { + opt.pluginExtender[pluginName] = e + } +} diff --git a/simulator/pkg/debuggablescheduler/debuggable_scheduler.go b/simulator/pkg/debuggablescheduler/debuggable_scheduler.go new file mode 100644 index 000000000..77518bddb --- /dev/null +++ b/simulator/pkg/debuggablescheduler/debuggable_scheduler.go @@ -0,0 +1,184 @@ +package debuggablescheduler + +import ( + "context" + "flag" + "os" + + "golang.org/x/xerrors" + clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + componentbaseconfig "k8s.io/component-base/config" + _ "k8s.io/component-base/logs/json/register" // for JSON log format registration + _ "k8s.io/component-base/metrics/prometheus/clientgo" + _ "k8s.io/component-base/metrics/prometheus/version" // for version metric registration + "k8s.io/klog/v2" + v1 "k8s.io/kube-scheduler/config/v1" + "k8s.io/kubernetes/cmd/kube-scheduler/app" + "k8s.io/kubernetes/pkg/scheduler/apis/config" + "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" + configv1 "k8s.io/kubernetes/pkg/scheduler/apis/config/v1" + configv1beta2 "k8s.io/kubernetes/pkg/scheduler/apis/config/v1beta2" + configv1beta3 "k8s.io/kubernetes/pkg/scheduler/apis/config/v1beta3" + "k8s.io/kubernetes/pkg/scheduler/framework/runtime" + + simulatorconfig "sigs.k8s.io/kube-scheduler-simulator/simulator/config" + "sigs.k8s.io/kube-scheduler-simulator/simulator/scheduler" + simulatorschedulerconfig "sigs.k8s.io/kube-scheduler-simulator/simulator/scheduler/config" + "sigs.k8s.io/kube-scheduler-simulator/simulator/scheduler/plugin" + "sigs.k8s.io/kube-scheduler-simulator/simulator/scheduler/storereflector" +) + +// CreateOptionForOutOfTreePlugin creates the option which can be help with running the external scheduler. +// It does: +// - create the wrapped plugin registries and return the registries as app.Option +// - initialize and start the store reflector. +// - the scheduler config conversion +// - reads the scheduling config passed from users (or use the default config) +// - converts it for enabling wrapped plugins +// - makes the defaulting func of the KubeSchedulerConfig always returning the converted one. We can let the scheduler use the converted configuration under any circumstances because the scheduler will always use this defaulting func to load the configuration. +// +//nolint:funlen,cyclop +func CreateOptionForOutOfTreePlugin(outOfTreePluginRegistry runtime.Registry, pluginExtender map[string]plugin.PluginExtenderInitializer) ([]app.Option, func(), error) { + if outOfTreePluginRegistry != nil { + simulatorschedulerconfig.SetOutOfTreeRegistries(outOfTreePluginRegistry) + } + + // flags defined in the upstream scheduler + configFile := flag.String("config", "", "") + master := flag.String("master", "", "") + flag.Parse() + + var versionedcfg *v1.KubeSchedulerConfiguration + var err error + if configFile == nil { + versionedcfg, err = simulatorschedulerconfig.DefaultSchedulerConfig() + if err != nil { + return nil, nil, xerrors.Errorf("get default scheduler config: %w", err) + } + } else { + versionedcfg, err = loadConfigFromFile(*configFile) + if err != nil { + return nil, nil, xerrors.Errorf("load scheduler config: %w", err) + } + } + + versioned, err := scheduler.ConvertConfigurationForSimulator(versionedcfg) + if err != nil { + return nil, nil, xerrors.Errorf("convert scheduler config to apply: %w", err) + } + + internalCfg, err := scheduler.ConvertSchedulerConfigToInternalConfig(versioned) + if err != nil { + return nil, nil, xerrors.Errorf("convert scheduler config to internal one: %w", err) + } + + sharedStore := storereflector.New() + + registry, err := plugin.NewRegistry(sharedStore, internalCfg, pluginExtender) + if err != nil { + return nil, nil, xerrors.Errorf("convert scheduler config to apply: %w", err) + } + kubeconfig, err := simulatorconfig.GetKubeClientConfig() + if err != nil { + return nil, nil, xerrors.Errorf("get kubeconfig: %w", err) + } + if internalCfg.ClientConnection.Kubeconfig != "" { + kubeconfig, err = createKubeConfig(internalCfg.ClientConnection, *master) + if err != nil { + return nil, nil, xerrors.Errorf("get kubeconfig specified in config: %w", err) + } + } + clientSet, err := clientset.NewForConfig(kubeconfig) + if err != nil { + return nil, nil, xerrors.Errorf("creates a new Clientset for kubeconfig: %w", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + if err := sharedStore.ResisterResultSavingToInformer(clientSet, ctx.Done()); err != nil { + return nil, cancel, xerrors.Errorf("ResisterResultSavingToInformer of sharedStore: %w", err) + } + + // black magic: We need to use the scheduler config converted for the simulator in the external scheduler. + // Here, we overwrite the defaulting func for KubeSchedulerConfiguration, + // so that user's config will be replaced with the one we created here + // when the scheduler loads the scheduler config + // or when loading the default scheduler config. + scheme.Scheme.AddTypeDefaultingFunc(&v1.KubeSchedulerConfiguration{}, func(obj interface{}) { + c, ok := obj.(*v1.KubeSchedulerConfiguration) + if !ok { + panic("unexpected type") + } + configv1.SetObjectDefaults_KubeSchedulerConfiguration(c) + c.Profiles = versioned.Profiles + }) + + return generateWithPluginOptions(registry), cancel, nil +} + +// createKubeConfig creates a kubeConfig from the given config and masterOverride. +func createKubeConfig(config componentbaseconfig.ClientConnectionConfiguration, masterOverride string) (*restclient.Config, error) { + kubeConfig, err := clientcmd.BuildConfigFromFlags(masterOverride, config.Kubeconfig) + if err != nil { + return nil, err + } + + kubeConfig.DisableCompression = true + kubeConfig.AcceptContentTypes = config.AcceptContentTypes + kubeConfig.ContentType = config.ContentType + kubeConfig.QPS = config.QPS + kubeConfig.Burst = int(config.Burst) + + return kubeConfig, nil +} + +func generateWithPluginOptions(registry map[string]runtime.PluginFactory) []app.Option { + opt := make([]app.Option, 0, len(registry)) + for k, r := range registry { + opt = append(opt, app.WithPlugin(k, r)) + } + return opt +} + +func loadConfigFromFile(file string) (*v1.KubeSchedulerConfiguration, error) { + data, err := os.ReadFile(file) + if err != nil { + return nil, err + } + + return loadConfig(data) +} + +func loadConfig(data []byte) (*v1.KubeSchedulerConfiguration, error) { + // The UniversalDecoder runs defaulting and returns the internal type by default. + obj, gvk, err := scheme.Codecs.UniversalDecoder().Decode(data, nil, nil) + if err != nil { + return nil, err + } + if cfgObj, ok := obj.(*config.KubeSchedulerConfiguration); ok { + // We don't set this field in pkg/scheduler/apis/config/{version}/conversion.go + // because the field will be cleared later by API machinery during + // conversion. See KubeSchedulerConfiguration internal type definition for + // more details. + cfgObj.TypeMeta.APIVersion = gvk.GroupVersion().String() + switch cfgObj.TypeMeta.APIVersion { + case configv1beta2.SchemeGroupVersion.String(): + klog.InfoS("KubeSchedulerConfiguration v1beta2 is deprecated in v1.25, will be removed in v1.28") + case configv1beta3.SchemeGroupVersion.String(): + klog.InfoS("KubeSchedulerConfiguration v1beta3 is deprecated in v1.26, will be removed in v1.29") + } + + return convertSchedulerConfigToV1Config(cfgObj) + } + return nil, xerrors.Errorf("couldn't decode as KubeSchedulerConfiguration, got %s", gvk) +} + +func convertSchedulerConfigToV1Config(versioned *config.KubeSchedulerConfiguration) (*v1.KubeSchedulerConfiguration, error) { + cfg := v1.KubeSchedulerConfiguration{} + if err := scheme.Scheme.Convert(versioned, &cfg, nil); err != nil { + return nil, xerrors.Errorf("convert configuration: %w", err) + } + + return &cfg, nil +} diff --git a/simulator/pkg/externalscheduler/command.go b/simulator/pkg/externalscheduler/command.go index 3739f9a9f..f33104ec2 100644 --- a/simulator/pkg/externalscheduler/command.go +++ b/simulator/pkg/externalscheduler/command.go @@ -1,3 +1,4 @@ +// Deprecated: it was renamed to the debuggablescheduler plugin. We'll remove it soon. package externalscheduler import ( @@ -8,6 +9,7 @@ import ( "sigs.k8s.io/kube-scheduler-simulator/simulator/scheduler/plugin" ) +// Deprecated: the externalscheduler package was renamed to the debuggablescheduler plugin. We'll remove it soon. func NewSchedulerCommand(opts ...Option) (*cobra.Command, func(), error) { opt := &options{pluginExtender: map[string]plugin.PluginExtenderInitializer{}, outOfTreeRegistry: map[string]runtime.PluginFactory{}} for _, o := range opts { @@ -29,9 +31,11 @@ type options struct { pluginExtender map[string]plugin.PluginExtenderInitializer } +// Deprecated: the externalscheduler package was renamed to the debuggablescheduler plugin. We'll remove it soon. type Option func(opt *options) // WithPlugin creates an Option based on plugin name and factory. +// Deprecated: the externalscheduler package was renamed to the debuggablescheduler plugin. We'll remove it soon. func WithPlugin(pluginName string, factory runtime.PluginFactory) Option { return func(opt *options) { opt.outOfTreeRegistry[pluginName] = factory @@ -39,6 +43,7 @@ func WithPlugin(pluginName string, factory runtime.PluginFactory) Option { } // WithPluginExtenders creates an Option based on plugin name and plugin extenders. +// Deprecated: the externalscheduler package was renamed to the debuggablescheduler plugin. We'll remove it soon. func WithPluginExtenders(pluginName string, e plugin.PluginExtenderInitializer) Option { return func(opt *options) { opt.pluginExtender[pluginName] = e diff --git a/simulator/pkg/externalscheduler/external_scheduler.go b/simulator/pkg/externalscheduler/external_scheduler.go index 6bc3867b8..c875f9478 100644 --- a/simulator/pkg/externalscheduler/external_scheduler.go +++ b/simulator/pkg/externalscheduler/external_scheduler.go @@ -30,7 +30,7 @@ import ( "sigs.k8s.io/kube-scheduler-simulator/simulator/scheduler/storereflector" ) -// CreateOptionForOutOfTreePlugin creates the option which can be help with running the external scheduler. +// CreateOptionForOutOfTreePlugin creates the option which can be help with running the debuggable scheduler. // It does: // - create the wrapped plugin registries and return the registries as app.Option // - initialize and start the store reflector. @@ -39,6 +39,8 @@ import ( // - converts it for enabling wrapped plugins // - makes the defaulting func of the KubeSchedulerConfig always returning the converted one. We can let the scheduler use the converted configuration under any circumstances because the scheduler will always use this defaulting func to load the configuration. // +// Deprecated: the externalscheduler package was renamed to the debuggablescheduler plugin. We'll remove it soon. +// //nolint:funlen,cyclop func CreateOptionForOutOfTreePlugin(outOfTreePluginRegistry runtime.Registry, pluginExtender map[string]plugin.PluginExtenderInitializer) ([]app.Option, func(), error) { if outOfTreePluginRegistry != nil { @@ -100,7 +102,7 @@ func CreateOptionForOutOfTreePlugin(outOfTreePluginRegistry runtime.Registry, pl return nil, cancel, xerrors.Errorf("ResisterResultSavingToInformer of sharedStore: %w", err) } - // black magic: We need to use the scheduler config converted for the simulator in the external scheduler. + // black magic: We need to use the scheduler config converted for the simulator in the debuggable scheduler. // Here, we overwrite the defaulting func for KubeSchedulerConfiguration, // so that user's config will be replaced with the one we created here // when the scheduler loads the scheduler config