doc: create the doc for the debuggable scheduler

README.md

@@ -41,15 +41,15 @@ We have several components:
 
 ### Simulator
 
-Simulator is kube-apiserver + scheduler + the HTTP server which mainly for the web UI.
+Simulator is kube-apiserver with some controllers + debuggable scheduler + the HTTP server which mainly for the web UI.
 
 There are several ways to integrate your scheduler into the simulator.
 See [integrate-your-scheduler.md](simulator/docs/integrate-your-scheduler.md).
 
-You can create any resources by communicating with kube-apiserver in any ways (kubectl, k8s client library, or web UI described next)
-and see how your Pods are scheduled.
-
-When you create Pods, Pods will get annotations from the simulator which contains the scheduling results per plugins or extenders.
+You can create any resources by any ways (kubectl, k8s client library, or web UI described next).
+And when you create Pods, 
+Pods will be scheduled by the [debuggable scheduler](./simulator/docs/debuggable-scheduler.md),
+and they'll get the annotations that explain how each Pod was evaluated by each scheduler plugin.
 
 ```yaml
 kind: Pod
@@ -81,7 +81,8 @@ metadata:
 
 You can utilize these results to understand your scheduler, check/test your configurations or customized scheduler's behavior.
 
-Further expansion, you can export internal state more, change specific behaviours on plugins etc by implementing [PluginExtender](./simulator/docs/plugin-extender.md).
+Further expansion, you can export internal state more, or change specific behaviours on plugins 
+by implementing [PluginExtender](./simulator/docs/plugin-extender.md).
 
 The simulator has its own configuration,
 you can refer to the [documentation](./simulator/docs/simulator-server-config.md).

simulator/docs/debuggable-scheduler.md

@@ -0,0 +1,94 @@
+## Debuggable scheduler
+
+The "debuggable scheduler" is the core concept of the simulator.
+
+### What's the "debuggable scheduler"?
+
+The debuggable scheduler is the scheduler that schedules Pods like other schedulers, 
+but it exposes the scheduling results from each scheduling plugin on Pod annotation.
+
+```yaml
+kind: Pod
+apiVersion: v1
+metadata:
+  name: hoge-pod
+  annotations:
+    scheduler-simulator/bind-result: '{"DefaultBinder":"success"}'
+    scheduler-simulator/filter-result: >-
+      {"node-282x7":{"AzureDiskLimits":"passed","EBSLimits":"passed","GCEPDLimits":"passed","InterPodAffinity":"passed","NodeAffinity":"passed","NodeName":"passed","NodePorts":"passed","NodeResourcesFit":"passed","NodeUnschedulable":"passed","NodeVolumeLimits":"passed","PodTopologySpread":"passed","TaintToleration":"passed","VolumeBinding":"passed","VolumeRestrictions":"passed","VolumeZone":"passed"},"node-gp9t4":{"AzureDiskLimits":"passed","EBSLimits":"passed","GCEPDLimits":"passed","InterPodAffinity":"passed","NodeAffinity":"passed","NodeName":"passed","NodePorts":"passed","NodeResourcesFit":"passed","NodeUnschedulable":"passed","NodeVolumeLimits":"passed","PodTopologySpread":"passed","TaintToleration":"passed","VolumeBinding":"passed","VolumeRestrictions":"passed","VolumeZone":"passed"}}
+    scheduler-simulator/finalscore-result: >-
+      {"node-282x7":{"ImageLocality":"0","InterPodAffinity":"0","NodeAffinity":"0","NodeNumber":"0","NodeResourcesBalancedAllocation":"76","NodeResourcesFit":"73","PodTopologySpread":"200","TaintToleration":"300","VolumeBinding":"0"},"node-gp9t4":{"ImageLocality":"0","InterPodAffinity":"0","NodeAffinity":"0","NodeNumber":"0","NodeResourcesBalancedAllocation":"76","NodeResourcesFit":"73","PodTopologySpread":"200","TaintToleration":"300","VolumeBinding":"0"}}
+    scheduler-simulator/permit-result: '{}'
+    scheduler-simulator/permit-result-timeout: '{}'
+    scheduler-simulator/postfilter-result: '{}'
+    scheduler-simulator/prebind-result: '{"VolumeBinding":"success"}'
+    scheduler-simulator/prefilter-result: '{}'
+    scheduler-simulator/prefilter-result-status: >-
+      {"InterPodAffinity":"success","NodeAffinity":"success","NodePorts":"success","NodeResourcesFit":"success","PodTopologySpread":"success","VolumeBinding":"success","VolumeRestrictions":"success"}
+    scheduler-simulator/prescore-result: >-
+      {"InterPodAffinity":"success","NodeAffinity":"success","NodeNumber":"success","PodTopologySpread":"success","TaintToleration":"success"}
+    scheduler-simulator/reserve-result: '{"VolumeBinding":"success"}'
+    scheduler-simulator/result-history: >-
+      [{"noderesourcefit-prefilter-data":"{\"MilliCPU\":100,\"Memory\":17179869184,\"EphemeralStorage\":0,\"AllowedPodNumber\":0,\"ScalarResources\":null}","scheduler-simulator/bind-result":"{\"DefaultBinder\":\"success\"}","scheduler-simulator/filter-result":"{\"node-282x7\":{\"AzureDiskLimits\":\"passed\",\"EBSLimits\":\"passed\",\"GCEPDLimits\":\"passed\",\"InterPodAffinity\":\"passed\",\"NodeAffinity\":\"passed\",\"NodeName\":\"passed\",\"NodePorts\":\"passed\",\"NodeResourcesFit\":\"passed\",\"NodeUnschedulable\":\"passed\",\"NodeVolumeLimits\":\"passed\",\"PodTopologySpread\":\"passed\",\"TaintToleration\":\"passed\",\"VolumeBinding\":\"passed\",\"VolumeRestrictions\":\"passed\",\"VolumeZone\":\"passed\"},\"node-gp9t4\":{\"AzureDiskLimits\":\"passed\",\"EBSLimits\":\"passed\",\"GCEPDLimits\":\"passed\",\"InterPodAffinity\":\"passed\",\"NodeAffinity\":\"passed\",\"NodeName\":\"passed\",\"NodePorts\":\"passed\",\"NodeResourcesFit\":\"passed\",\"NodeUnschedulable\":\"passed\",\"NodeVolumeLimits\":\"passed\",\"PodTopologySpread\":\"passed\",\"TaintToleration\":\"passed\",\"VolumeBinding\":\"passed\",\"VolumeRestrictions\":\"passed\",\"VolumeZone\":\"passed\"}}","scheduler-simulator/finalscore-result":"{\"node-282x7\":{\"ImageLocality\":\"0\",\"InterPodAffinity\":\"0\",\"NodeAffinity\":\"0\",\"NodeNumber\":\"0\",\"NodeResourcesBalancedAllocation\":\"76\",\"NodeResourcesFit\":\"73\",\"PodTopologySpread\":\"200\",\"TaintToleration\":\"300\",\"VolumeBinding\":\"0\"},\"node-gp9t4\":{\"ImageLocality\":\"0\",\"InterPodAffinity\":\"0\",\"NodeAffinity\":\"0\",\"NodeNumber\":\"0\",\"NodeResourcesBalancedAllocation\":\"76\",\"NodeResourcesFit\":\"73\",\"PodTopologySpread\":\"200\",\"TaintToleration\":\"300\",\"VolumeBinding\":\"0\"}}","scheduler-simulator/permit-result":"{}","scheduler-simulator/permit-result-timeout":"{}","scheduler-simulator/postfilter-result":"{}","scheduler-simulator/prebind-result":"{\"VolumeBinding\":\"success\"}","scheduler-simulator/prefilter-result":"{}","scheduler-simulator/prefilter-result-status":"{\"InterPodAffinity\":\"success\",\"NodeAffinity\":\"success\",\"NodePorts\":\"success\",\"NodeResourcesFit\":\"success\",\"PodTopologySpread\":\"success\",\"VolumeBinding\":\"success\",\"VolumeRestrictions\":\"success\"}","scheduler-simulator/prescore-result":"{\"InterPodAffinity\":\"success\",\"NodeAffinity\":\"success\",\"NodeNumber\":\"success\",\"PodTopologySpread\":\"success\",\"TaintToleration\":\"success\"}","scheduler-simulator/reserve-result":"{\"VolumeBinding\":\"success\"}","scheduler-simulator/score-result":"{\"node-282x7\":{\"ImageLocality\":\"0\",\"InterPodAffinity\":\"0\",\"NodeAffinity\":\"0\",\"NodeNumber\":\"0\",\"NodeResourcesBalancedAllocation\":\"76\",\"NodeResourcesFit\":\"73\",\"PodTopologySpread\":\"0\",\"TaintToleration\":\"0\",\"VolumeBinding\":\"0\"},\"node-gp9t4\":{\"ImageLocality\":\"0\",\"InterPodAffinity\":\"0\",\"NodeAffinity\":\"0\",\"NodeNumber\":\"0\",\"NodeResourcesBalancedAllocation\":\"76\",\"NodeResourcesFit\":\"73\",\"PodTopologySpread\":\"0\",\"TaintToleration\":\"0\",\"VolumeBinding\":\"0\"}}","scheduler-simulator/selected-node":"node-282x7"}]
+    scheduler-simulator/score-result: >-
+      {"node-282x7":{"ImageLocality":"0","InterPodAffinity":"0","NodeAffinity":"0","NodeNumber":"0","NodeResourcesBalancedAllocation":"76","NodeResourcesFit":"73","PodTopologySpread":"0","TaintToleration":"0","VolumeBinding":"0"},"node-gp9t4":{"ImageLocality":"0","InterPodAffinity":"0","NodeAffinity":"0","NodeNumber":"0","NodeResourcesBalancedAllocation":"76","NodeResourcesFit":"73","PodTopologySpread":"0","TaintToleration":"0","VolumeBinding":"0"}}
+    scheduler-simulator/selected-node: node-282x7
+```
+
+The simulator runs the debuggable scheduler built from the upstream scheduler by default,
+that's why we can see the scheduling results like the above in the simulator.
+
+## Build a debuggable scheduler from your scheduler
+
+You can use the [`debuggablescheduler` package](../pkg/debuggablescheduler) to transform your scheduler into a debuggable scheduler.
+
+The debuggable scheduler could work anywhere, in the simulator or even in your cluster.
+
+### Change your scheduler
+
+Here, we assume you're registering your custom plugins in your scheduler like this:
+
+```go
+// your scheduler's main package
+func main() {
+	command := app.NewSchedulerCommand(
+		app.WithPlugin(yourcustomplugin.Name, yourcustomplugin.New),
+	)
+
+	code := cli.Run(command)
+	os.Exit(code)
+}
+```
+
+Then, you need to replace few lines to use the [`debuggablescheduler` package](../pkg/debuggablescheduler).
+
+```go
+func main() {
+	command, cancelFn, err := debuggablescheduler.NewSchedulerCommand(
+        debuggablescheduler.WithPlugin(yourcustomplugin.Name, yourcustomplugin.New),
+        debuggablescheduler.WithPluginExtenders(noderesources.Name, extender.New), // [optional] see plugin-extender.md about PluginExtender.
+    )
+    if err != nil {
+        klog.Info(fmt.Sprintf("failed to build the scheduler command: %+v", err))
+        os.Exit(1)
+    }
+    code := cli.Run(command)
+    cancelFn()
+    os.Exit(code)
+}
+```
+
+As you see, `debuggablescheduler.NewSchedulerCommand` has much similar interface as the `app.NewSchedulerCommand`.
+You can register your plugins by `debuggablescheduler.WithPlugin` option.
+
+### The plugin extender
+
+We have the plugin extender feature to expand the debuggable scheduler more.
+See [plugin-extender.md](./plugin-extender.md) to know more about it.
+
+### The example debuggable scheduler
+
+We have the sample implementation in [./sample/debuggable-scheduler](./sample/debuggable-scheduler).
+
+You can try to run it in the simulator via [external scheduler](./external-scheduler.md) feature.
+See [external-scheduler.md#the-example-external-scheduler](./external-scheduler.md#the-example-external-scheduler).

simulator/docs/external-scheduler.md

@@ -1,67 +1,16 @@
 ## External scheduler
 
-This document describes about the feature called "external scheduler"
+The external scheduler is the feature of the simulator 
+to allow you to run any of your scheduler in the simulator instead of the default one.
 
-We use the [`externalscheduler` package](../pkg/externalscheduler); 
-the scheduler built with the [`externalscheduler` package](../pkg/externalscheduler) will export the scheduling results on each Pod annotation.
+Before diving into the description, you need to know [debuggable scheduler](./debuggable-scheduler.md).
 
-### Use cases
+### connect your scheduler to the simulator
 
-- Running your scheduler instead of the default one in the simulator 
-  - You can still see the scheduling results in web UI as well!
-- Running your scheduler with the simulator's feature in your clusterr.
-  - Like in the simulator, all Pods will get the scheduling results on its annotation while each scheduling is done as usual. 
-  - Note that it has performance overhead in each scheduling cycle. 
-since the scheduler needs to make additional effort to export the scheduling results.
-
-### Change your scheduler
-
-Here, we assume you're registering your custom plugins in your scheduler like this:
-
-```go
-// your scheduler's main package
-func main() {
-	command := app.NewSchedulerCommand(
-		app.WithPlugin(yourcustomplugin.Name, yourcustomplugin.New),
-	)
-
-	code := cli.Run(command)
-	os.Exit(code)
-}
-```
-
-Then, you need to replace few lines to use the [`externalscheduler` package](../pkg/externalscheduler).
-
-```go
-func main() {
-	command, cancelFn, err := externalscheduler.NewSchedulerCommand(
-        externalscheduler.WithPlugin(yourcustomplugin.Name, yourcustomplugin.New),
-        externalscheduler.WithPluginExtenders(noderesources.Name, extender.New), // [optional] see plugin-extender.md about PluginExtender.
-    )
-    if err != nil {
-        klog.Info(fmt.Sprintf("failed to build the scheduler command: %+v", err))
-        os.Exit(1)
-    }
-    code := cli.Run(command)
-    cancelFn()
-    os.Exit(code)
-}
-```
-
-As you see, `externalscheduler.NewSchedulerCommand` has much similar interface as the `app.NewSchedulerCommand`.
-You can register your plugins by `externalscheduler.WithPlugin` option.
-
-Via this step, all Pods scheduled by this scheduler will get the scheduling results in the annotation like in the simulator!
-
-### Connect your scheduler to the kube-apiserver in the simulator
-
-If you are here to run the scheduler built with [`externalscheduler` package](../pkg/externalscheduler) in your cluster, 
-you don't need to follow this step.
-
-Let's connect your scheduler into the simulator.
+Let's connect your scheduler to the simulator.
 
 First, you need to set `externalSchedulerEnabled: true` on [the simulator config](../config.yaml)
-so that the scheduler in the simulator won't get started.
+so that the scheduler, running in the simulator by default, won't get started.
 
 Next, you need to connect your scheduler into the simulator's kube-apiserver via KubeSchedulerConfig:
 
@@ -76,14 +25,15 @@ You can use this [kubeconfig.yaml](./kubeconfig.yaml) to communicate with the si
 
 ### The example external scheduler
 
-We have the sample external scheduler implementation in [./sample/external-scheduler](./sample/external-scheduler).
+You can see how the external scheduler can be set up 
+with the sample debuggable scheduler implementation in [./sample/debuggable-scheduler](./sample/debuggable-scheduler).
 
 prerequisite:
 1. set `externalSchedulerEnabled: true` on [the simulator config](../config.yaml)
 2. run the simulator 
 
 ```shell
-cd sample/external-scheduler
+cd sample/debuggable-scheduler
 go run main.go --config scheduler.yaml
 ```
 

simulator/docs/plugin-extender.md

@@ -50,13 +50,13 @@ each Pod will get `"noderesourcefit-prefilter-data": prefilterData` annotation i
 
 **Currently, the plugin extender can be used only in [the external scheduler](./external-scheduler.md).**
 
-You can use `externalscheduler.WithPluginExtenders` option in `externalscheduler.NewSchedulerCommand`
+You can use `debuggablescheduler.WithPluginExtenders` option in `debuggablescheduler.NewSchedulerCommand`
 to enable some PluginExtender in particular plugin.
 
 ```go
 func main() {
-	command, cancelFn, err := externalscheduler.NewSchedulerCommand(
-        externalscheduler.WithPluginExtenders(noderesources.Name, extender.New),
+	command, cancelFn, err := debuggablescheduler.NewSchedulerCommand(
+        debuggablescheduler.WithPluginExtenders(noderesources.Name, extender.New),
     )
     if err != nil {
         klog.Info(fmt.Sprintf("failed to build the scheduler command: %+v", err))

simulator/docs/sample/external-scheduler/main.go → simulator/docs/sample/debuggable-scheduler/main.go

@@ -14,13 +14,13 @@ import (
 
 	"sigs.k8s.io/kube-scheduler-simulator/simulator/docs/sample/nodenumber"
 	extender "sigs.k8s.io/kube-scheduler-simulator/simulator/docs/sample/plugin-extender"
-	"sigs.k8s.io/kube-scheduler-simulator/simulator/pkg/externalscheduler"
+	"sigs.k8s.io/kube-scheduler-simulator/simulator/pkg/debuggablescheduler"
 )
 
 func main() {
-	command, cancelFn, err := externalscheduler.NewSchedulerCommand(
-		externalscheduler.WithPlugin(nodenumber.Name, nodenumber.New),
-		externalscheduler.WithPluginExtenders(noderesources.Name, extender.New),
+	command, cancelFn, err := debuggablescheduler.NewSchedulerCommand(
+		debuggablescheduler.WithPlugin(nodenumber.Name, nodenumber.New),           // Register the custom scheduler plugin.
+		debuggablescheduler.WithPluginExtenders(noderesources.Name, extender.New), // [Optional] Register the plugin extender. See /simulator/docs/plugin-extender.md
 	)
 	if err != nil {
 		klog.Info(fmt.Sprintf("failed to build the scheduler command: %+v", err))

simulator/pkg/debuggablescheduler/command.go

@@ -0,0 +1,46 @@
+package debuggablescheduler
+
+import (
+	"github.com/spf13/cobra"
+	"k8s.io/kubernetes/cmd/kube-scheduler/app"
+	"k8s.io/kubernetes/pkg/scheduler/framework/runtime"
+
+	"sigs.k8s.io/kube-scheduler-simulator/simulator/scheduler/plugin"
+)
+
+func NewSchedulerCommand(opts ...Option) (*cobra.Command, func(), error) {
+	opt := &options{pluginExtender: map[string]plugin.PluginExtenderInitializer{}, outOfTreeRegistry: map[string]runtime.PluginFactory{}}
+	for _, o := range opts {
+		o(opt)
+	}
+
+	scheduleropts, cancelFn, err := CreateOptionForOutOfTreePlugin(opt.outOfTreeRegistry, opt.pluginExtender)
+	if err != nil {
+		return nil, cancelFn, err
+	}
+
+	command := app.NewSchedulerCommand(scheduleropts...)
+
+	return command, cancelFn, nil
+}
+
+type options struct {
+	outOfTreeRegistry runtime.Registry
+	pluginExtender    map[string]plugin.PluginExtenderInitializer
+}
+
+type Option func(opt *options)
+
+// WithPlugin creates an Option based on plugin name and factory.
+func WithPlugin(pluginName string, factory runtime.PluginFactory) Option {
+	return func(opt *options) {
+		opt.outOfTreeRegistry[pluginName] = factory
+	}
+}
+
+// WithPluginExtenders creates an Option based on plugin name and plugin extenders.
+func WithPluginExtenders(pluginName string, e plugin.PluginExtenderInitializer) Option {
+	return func(opt *options) {
+		opt.pluginExtender[pluginName] = e
+	}
+}