Skip to content

Commit

Permalink
run kube controllers separately based on their command
Browse files Browse the repository at this point in the history
  • Loading branch information
deads2k committed Aug 31, 2017
1 parent 457a31c commit ba3276f
Show file tree
Hide file tree
Showing 4 changed files with 131 additions and 84 deletions.
39 changes: 0 additions & 39 deletions pkg/cmd/server/kubernetes/master/controller/config.go

This file was deleted.

102 changes: 102 additions & 0 deletions pkg/cmd/server/start/start_kube_controller_manager.go
Original file line number Diff line number Diff line change
@@ -1 +1,103 @@
package start

import (
"github.com/golang/glog"

kerrors "k8s.io/apimachinery/pkg/util/errors"
controllerapp "k8s.io/kubernetes/cmd/kube-controller-manager/app"
controlleroptions "k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
_ "k8s.io/kubernetes/plugin/pkg/scheduler/algorithmprovider"

"github.com/openshift/origin/pkg/cmd/server/bootstrappolicy"
"k8s.io/kubernetes/pkg/api/v1"
kapiv1 "k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/volume"

cmdflags "github.com/openshift/origin/pkg/cmd/util/flags"
"github.com/spf13/pflag"
)

// newPersistentVolumeRecyclerPodTemplate provides a function which makes our recycler pod template for use in the kube-controller-manager
// this is a stop-gap until the kube-controller-manager take a pod manifest
func newPersistentVolumeRecyclerPodTemplate(recyclerImageName string) func() *v1.Pod {
oldTemplateFunc := volume.NewPersistentVolumeRecyclerPodTemplate
return func() *v1.Pod {
uid := int64(0)
defaultScrubPod := oldTemplateFunc()
// TODO: Move the recycler pods to dedicated namespace instead of polluting openshift-infra.
defaultScrubPod.Namespace = "openshift-infra"
defaultScrubPod.Spec.ServiceAccountName = bootstrappolicy.InfraPersistentVolumeRecyclerControllerServiceAccountName
defaultScrubPod.Spec.Containers[0].Image = recyclerImageName
defaultScrubPod.Spec.Containers[0].Command = []string{"/usr/bin/openshift-recycle"}
defaultScrubPod.Spec.Containers[0].Args = []string{"/scrub"}
defaultScrubPod.Spec.Containers[0].SecurityContext = &kapiv1.SecurityContext{RunAsUser: &uid}
defaultScrubPod.Spec.Containers[0].ImagePullPolicy = kapiv1.PullIfNotPresent

return defaultScrubPod
}
}

func kubeControllerManagerAddFlags(cmserver *controlleroptions.CMServer) func(flags *pflag.FlagSet) {
return func(flags *pflag.FlagSet) {
cmserver.AddFlags(flags, controllerapp.KnownControllers(), controllerapp.ControllersDisabledByDefault.List())
}
}

func newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile string, cmdLineArgs map[string][]string) (*controlleroptions.CMServer, error) {
if cmdLineArgs == nil {
cmdLineArgs = map[string][]string{}
}

if len(cmdLineArgs["controllers"]) == 0 {
cmdLineArgs["controllers"] = []string{}
}
// these two are ones we disable in addition to others
cmdLineArgs["controllers"] = []string{
// we don't appear to use this
"-ttl",
// we have to configure this separately until it is generic
"-horizontalpodautoscaler",
// we carry patches on this. For now....
"-serviceaccount-token",
}
if len(cmdLineArgs["use-service-account-credentials"]) == 0 {
cmdLineArgs["use-service-account-credentials"] = []string{"true"}
}
if len(cmdLineArgs["service-account-private-key-file"]) == 0 {
cmdLineArgs["service-account-private-key-file"] = []string{saPrivateKeyFile}
}
if len(cmdLineArgs["root-ca-file"]) == 0 {
cmdLineArgs["root-ca-file"] = []string{saRootCAFile}
}
if len(cmdLineArgs["kubeconfig"]) == 0 {
cmdLineArgs["kubeconfig"] = []string{kubeconfigFile}
}

// disable serving http since we didn't used to expose it
if len(cmdLineArgs["port"]) == 0 {
cmdLineArgs["port"] = []string{"-1"}
}

// resolve arguments
controllerManager := controlleroptions.NewCMServer()
if err := cmdflags.Resolve(cmdLineArgs, kubeControllerManagerAddFlags(controllerManager)); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}

return controllerManager, nil
}

func runEmbeddedKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile string, cmdLineArgs map[string][]string, recyclerImage string) {
volume.NewPersistentVolumeRecyclerPodTemplate = newPersistentVolumeRecyclerPodTemplate(recyclerImage)

// TODO we need a real identity for this. Right now it's just using the loopback connection like it used to.
controllerManager, err := newKubeControllerManager(kubeconfigFile, saPrivateKeyFile, saRootCAFile, cmdLineArgs)
if err != nil {
glog.Fatal(err)
}
// this does a second leader election, but doing the second leader election will allow us to move out process in
// 3.8 if we so choose.
if err := controllerapp.Run(controllerManager); err != nil {
glog.Fatal(err)
}
}
34 changes: 8 additions & 26 deletions pkg/cmd/server/start/start_master.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import (
clientgoclientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
aggregatorinstall "k8s.io/kube-aggregator/pkg/apis/apiregistration/install"
kubecontroller "k8s.io/kubernetes/cmd/kube-controller-manager/app"
kapi "k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/capabilities"
kinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/externalversions"
Expand All @@ -47,7 +46,6 @@ import (
"github.com/openshift/origin/pkg/cmd/server/etcd"
"github.com/openshift/origin/pkg/cmd/server/etcd/etcdserver"
kubernetes "github.com/openshift/origin/pkg/cmd/server/kubernetes/master"
kubecontrollers "github.com/openshift/origin/pkg/cmd/server/kubernetes/master/controller"
"github.com/openshift/origin/pkg/cmd/server/origin"
origincontrollers "github.com/openshift/origin/pkg/cmd/server/origin/controller"
originrest "github.com/openshift/origin/pkg/cmd/server/origin/rest"
Expand Down Expand Up @@ -407,11 +405,13 @@ func (m *Master) Start() error {
return err
}

imageTemplate := variable.NewDefaultImageTemplate()
imageTemplate.Format = m.config.ImageConfig.Format
imageTemplate.Latest = m.config.ImageConfig.Latest
recyclerImage := imageTemplate.ExpandOrDie("recycler")

// you can't double run healthz, so only do this next bit if we aren't starting the API
if !m.api {
imageTemplate := variable.NewDefaultImageTemplate()
imageTemplate.Format = m.config.ImageConfig.Format
imageTemplate.Latest = m.config.ImageConfig.Latest

glog.Infof("Starting controllers on %s (%s)", m.config.ServingInfo.BindAddress, version.Get().String())
if len(m.config.DisabledFeatures) > 0 {
Expand Down Expand Up @@ -476,6 +476,8 @@ func (m *Master) Start() error {
// continuously run the scheduler while we have the primary lease
go runEmbeddedScheduler(m.config.MasterClients.OpenShiftLoopbackKubeConfig, m.config.KubernetesMasterConfig.SchedulerConfigFile, m.config.KubernetesMasterConfig.SchedulerArguments)

go runEmbeddedKubeControllerManager(m.config.MasterClients.OpenShiftLoopbackKubeConfig, m.config.ServiceAccountConfig.PrivateKeyFile, m.config.ServiceAccountConfig.MasterCA, m.config.KubernetesMasterConfig.ControllerArguments, recyclerImage)

controllerContext, err := getControllerContext(*m.config, kubeControllerManagerConfig, cloudProvider, informers, utilwait.NeverStop)
if err != nil {
glog.Fatal(err)
Expand Down Expand Up @@ -700,24 +702,10 @@ func startControllers(options configapi.MasterConfig, allocationController origi

allocationController.RunSecurityAllocationController()

// set the upstream default until it is configurable
kubecontrollers.SetPVRecyclerPod(options.ImageConfig)
kubernetesControllerInitializers := kubecontroller.NewControllerInitializers()
// remove the HPA controller until it is generic
delete(kubernetesControllerInitializers, "horizontalpodautoscaling")

openshiftControllerInitializers, err := openshiftControllerConfig.GetControllerInitializers()
if err != nil {
return err
}
// Add kubernetes controllers initialized from Origin
for name, initFn := range kubernetesControllerInitializers {
if _, exists := openshiftControllerInitializers[name]; exists {
// don't overwrite, openshift takes priority
continue
}
openshiftControllerInitializers[name] = origincontrollers.FromKubeInitFunc(initFn)
}

excludedControllers := getExcludedControllers(options)

Expand Down Expand Up @@ -751,17 +739,11 @@ func startControllers(options configapi.MasterConfig, allocationController origi
}

func getExcludedControllers(options configapi.MasterConfig) sets.String {
excludedControllers := sets.NewString(
// not used in openshift. Yet?
"ttl",
"bootstrapsigner",
"tokencleaner",
)
excludedControllers := sets.NewString()
if !configapi.IsBuildEnabled(&options) {
excludedControllers.Insert("openshift.io/build")
excludedControllers.Insert("openshift.io/build-config-change")
}

return excludedControllers
}

Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit ba3276f

Please sign in to comment.