Skip to content

Commit

Permalink
Ensure openshift start network can run in a pod
Browse files Browse the repository at this point in the history
Need to be able to take node-config from bootstrap node. For
openshift start network the --kubeconfig flag from the CLI overrides the
value of masterKubeConfig in the provided node config. If the value is
empty (like it is by default) the in-cluster-config is used.

Reorganize the node startup slightly so there is even less overlap
between kubelet and network. A future change will completely separate
these two initialization paths.
  • Loading branch information
smarterclayton committed Oct 11, 2017
1 parent ac98451 commit d349f9b
Show file tree
Hide file tree
Showing 9 changed files with 166 additions and 99 deletions.
2 changes: 2 additions & 0 deletions contrib/completions/bash/openshift
Original file line number Diff line number Diff line change
Expand Up @@ -33708,6 +33708,8 @@ _openshift_start_network()
local_nonpersistent_flags+=("--kubernetes=")
flags+=("--latest-images")
local_nonpersistent_flags+=("--latest-images")
flags+=("--listen=")
local_nonpersistent_flags+=("--listen=")
flags+=("--network-plugin=")
local_nonpersistent_flags+=("--network-plugin=")
flags+=("--recursive-resolv-conf=")
Expand Down
2 changes: 2 additions & 0 deletions contrib/completions/zsh/openshift
Original file line number Diff line number Diff line change
Expand Up @@ -33857,6 +33857,8 @@ _openshift_start_network()
local_nonpersistent_flags+=("--kubernetes=")
flags+=("--latest-images")
local_nonpersistent_flags+=("--latest-images")
flags+=("--listen=")
local_nonpersistent_flags+=("--listen=")
flags+=("--network-plugin=")
local_nonpersistent_flags+=("--network-plugin=")
flags+=("--recursive-resolv-conf=")
Expand Down
21 changes: 21 additions & 0 deletions pkg/cmd/server/api/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -334,6 +334,27 @@ func SetProtobufClientDefaults(overrides *ClientConnectionOverrides) {
overrides.Burst *= 2
}

// GetKubeConfigOrInClusterConfig loads in-cluster config if kubeConfigFile is empty or the file if not,
// then applies overrides.
func GetKubeConfigOrInClusterConfig(kubeConfigFile string, overrides *ClientConnectionOverrides) (*restclient.Config, error) {
var kubeConfig *restclient.Config
var err error
if len(kubeConfigFile) == 0 {
kubeConfig, err = restclient.InClusterConfig()
} else {
loadingRules := &clientcmd.ClientConfigLoadingRules{}
loadingRules.ExplicitPath = kubeConfigFile
loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})

kubeConfig, err = loader.ClientConfig()
}
if err != nil {
return nil, err
}
applyClientConnectionOverrides(overrides, kubeConfig)
return kubeConfig, nil
}

// TODO: clients should be copied and instantiated from a common client config, tweaked, then
// given to individual controllers and other infrastructure components.
func GetInternalKubeClient(kubeConfigFile string, overrides *ClientConnectionOverrides) (kclientsetinternal.Interface, *restclient.Config, error) {
Expand Down
2 changes: 1 addition & 1 deletion pkg/cmd/server/api/validation/validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ func ValidateServingInfo(info api.ServingInfo, certificatesRequired bool, fldPat
validationResults.AddErrors(ValidateFile(info.ClientCA, fldPath.Child("clientCA"))...)
}
} else {
if len(info.ClientCA) > 0 {
if certificatesRequired && len(info.ClientCA) > 0 {
validationResults.AddErrors(field.Invalid(fldPath.Child("clientCA"), info.ClientCA, "cannot specify a clientCA without a certFile"))
}
}
Expand Down
12 changes: 7 additions & 5 deletions pkg/cmd/server/kubernetes/network/network_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,12 @@ import (
"fmt"
"net"

"github.com/golang/glog"

miekgdns "github.com/miekg/dns"

kclientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/pkg/apis/componentconfig"
kclientsetexternal "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
kclientsetinternal "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
kinternalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"

configapi "github.com/openshift/origin/pkg/cmd/server/api"
Expand Down Expand Up @@ -45,11 +44,15 @@ type NetworkConfig struct {

// New creates a new network config object for running the networking components of the OpenShift node.
func New(options configapi.NodeConfig, clusterDomain string, proxyConfig *componentconfig.KubeProxyConfiguration, enableProxy, enableDNS bool) (*NetworkConfig, error) {
internalKubeClient, kubeConfig, err := configapi.GetInternalKubeClient(options.MasterKubeConfig, options.MasterClientConnectionOverrides)
kubeConfig, err := configapi.GetKubeConfigOrInClusterConfig(options.MasterKubeConfig, options.MasterClientConnectionOverrides)
if err != nil {
return nil, err
}
internalKubeClient, err := kclientsetinternal.NewForConfig(kubeConfig)
if err != nil {
return nil, err
}
externalKubeClient, _, err := configapi.GetExternalKubeClient(options.MasterKubeConfig, options.MasterClientConnectionOverrides)
externalKubeClient, err := kclientsetexternal.NewForConfig(kubeConfig)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -127,7 +130,6 @@ func New(options configapi.NodeConfig, clusterDomain string, proxyConfig *compon

// TODO: use kubeletConfig.ResolverConfig as an argument to etcd in the event the
// user sets it, instead of passing it to the kubelet.
glog.Infof("DNS Bind to %s", options.DNSBindAddress)
config.DNSServer = dns.NewServer(
dnsConfig,
services,
Expand Down
89 changes: 89 additions & 0 deletions pkg/cmd/server/kubernetes/network/options/options.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
package node

import (
"fmt"
"net"
"time"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kerrors "k8s.io/apimachinery/pkg/util/errors"
kubeproxyoptions "k8s.io/kubernetes/cmd/kube-proxy/app"
"k8s.io/kubernetes/pkg/apis/componentconfig"

configapi "github.com/openshift/origin/pkg/cmd/server/api"
cmdflags "github.com/openshift/origin/pkg/cmd/util/flags"
)

// Build creates the network Kubernetes component configs for a given NodeConfig, or returns
// an error
func Build(options configapi.NodeConfig) (*componentconfig.KubeProxyConfiguration, error) {
proxyOptions, err := kubeproxyoptions.NewOptions()
if err != nil {
return nil, err
}
// get default config
proxyconfig := proxyOptions.GetConfig()

proxyconfig.HostnameOverride = options.NodeName

// BindAddress - Override default bind address from our config
addr := options.ServingInfo.BindAddress
host, _, err := net.SplitHostPort(addr)
if err != nil {
return nil, fmt.Errorf("The provided value to bind to must be an ip:port %q", addr)
}
ip := net.ParseIP(host)
if ip == nil {
return nil, fmt.Errorf("The provided value to bind to must be an ip:port: %q", addr)
}
proxyconfig.BindAddress = ip.String()
// MetricsBindAddress - disable by default but allow enablement until we switch to
// reading proxy config directly
proxyconfig.MetricsBindAddress = ""
if arg := options.ProxyArguments["metrics-bind-address"]; len(arg) > 0 {
proxyconfig.MetricsBindAddress = arg[0]
}
delete(options.ProxyArguments, "metrics-bind-address")

// OOMScoreAdj, ResourceContainer - clear, we don't run in a container
oomScoreAdj := int32(0)
proxyconfig.OOMScoreAdj = &oomScoreAdj
proxyconfig.ResourceContainer = ""

// use the same client as the node
proxyconfig.ClientConnection.KubeConfigFile = options.MasterKubeConfig

// ProxyMode, set to iptables
proxyconfig.Mode = "iptables"

// IptablesSyncPeriod, set to our config value
syncPeriod, err := time.ParseDuration(options.IPTablesSyncPeriod)
if err != nil {
return nil, fmt.Errorf("Cannot parse the provided ip-tables sync period (%s) : %v", options.IPTablesSyncPeriod, err)
}
proxyconfig.IPTables.SyncPeriod = metav1.Duration{
Duration: syncPeriod,
}
masqueradeBit := int32(0)
proxyconfig.IPTables.MasqueradeBit = &masqueradeBit

// PortRange, use default
// HostnameOverride, use default
// ConfigSyncPeriod, use default
// MasqueradeAll, use default
// CleanupAndExit, use default
// KubeAPIQPS, use default, doesn't apply until we build a separate client
// KubeAPIBurst, use default, doesn't apply until we build a separate client
// UDPIdleTimeout, use default

// Resolve cmd flags to add any user overrides
if err := cmdflags.Resolve(options.ProxyArguments, proxyOptions.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}

if err := proxyOptions.Complete(); err != nil {
return nil, err
}

return proxyconfig, nil
}
3 changes: 2 additions & 1 deletion pkg/cmd/server/kubernetes/node/node_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
kubeletapp "k8s.io/kubernetes/cmd/kubelet/app"
kubeletoptions "k8s.io/kubernetes/cmd/kubelet/app/options"
"k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
kclientsetexternal "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
"k8s.io/kubernetes/pkg/cloudprovider"
"k8s.io/kubernetes/pkg/kubelet"
dockertools "k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker"
Expand Down Expand Up @@ -57,7 +58,7 @@ func New(options configapi.NodeConfig, server *kubeletoptions.KubeletServer) (*N
return nil, err
}
// Make a separate client for event reporting, to avoid event QPS blocking node calls
eventClient, _, err := configapi.GetExternalKubeClient(options.MasterKubeConfig, options.MasterClientConnectionOverrides)
eventClient, err := kclientsetexternal.NewForConfig(kubeConfig)
if err != nil {
return nil, err
}
Expand Down
87 changes: 8 additions & 79 deletions pkg/cmd/server/kubernetes/node/options/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kerrors "k8s.io/apimachinery/pkg/util/errors"
utilfeature "k8s.io/apiserver/pkg/util/feature"
kubeproxyoptions "k8s.io/kubernetes/cmd/kube-proxy/app"
kubeletoptions "k8s.io/kubernetes/cmd/kubelet/app/options"
"k8s.io/kubernetes/pkg/apis/componentconfig"
"k8s.io/kubernetes/pkg/features"
Expand All @@ -25,7 +24,7 @@ import (

// Build creates the core Kubernetes component configs for a given NodeConfig, or returns
// an error
func Build(options configapi.NodeConfig) (*kubeletoptions.KubeletServer, *componentconfig.KubeProxyConfiguration, error) {
func Build(options configapi.NodeConfig) (*kubeletoptions.KubeletServer, error) {
imageTemplate := variable.NewDefaultImageTemplate()
imageTemplate.Format = options.ImageConfig.Format
imageTemplate.Latest = options.ImageConfig.Latest
Expand All @@ -39,11 +38,11 @@ func Build(options configapi.NodeConfig) (*kubeletoptions.KubeletServer, *compon

kubeAddressStr, kubePortStr, err := net.SplitHostPort(options.ServingInfo.BindAddress)
if err != nil {
return nil, nil, fmt.Errorf("cannot parse node address: %v", err)
return nil, fmt.Errorf("cannot parse node address: %v", err)
}
kubePort, err := strconv.Atoi(kubePortStr)
if err != nil {
return nil, nil, fmt.Errorf("cannot parse node port: %v", err)
return nil, fmt.Errorf("cannot parse node port: %v", err)
}

// Defaults are tested in TestKubeletDefaults
Expand Down Expand Up @@ -91,7 +90,7 @@ func Build(options configapi.NodeConfig) (*kubeletoptions.KubeletServer, *compon
// Setup auth
authnTTL, err := time.ParseDuration(options.AuthConfig.AuthenticationCacheTTL)
if err != nil {
return nil, nil, err
return nil, err
}
server.Authentication = componentconfig.KubeletAuthentication{
X509: componentconfig.KubeletX509Authentication{
Expand All @@ -107,7 +106,7 @@ func Build(options configapi.NodeConfig) (*kubeletoptions.KubeletServer, *compon
}
authzTTL, err := time.ParseDuration(options.AuthConfig.AuthorizationCacheTTL)
if err != nil {
return nil, nil, err
return nil, err
}
server.Authorization = componentconfig.KubeletAuthorization{
Mode: componentconfig.KubeletAuthorizationModeWebhook,
Expand All @@ -121,13 +120,13 @@ func Build(options configapi.NodeConfig) (*kubeletoptions.KubeletServer, *compon
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubeletArguments, server.AddFlags); len(err) > 0 {
return nil, nil, kerrors.NewAggregate(err)
return nil, kerrors.NewAggregate(err)
}

// terminate early if feature gate is incorrect on the node
if len(server.FeatureGates) > 0 {
if err := utilfeature.DefaultFeatureGate.Set(server.FeatureGates); err != nil {
return nil, nil, err
return nil, err
}
}
if utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletServerCertificate) {
Expand All @@ -138,11 +137,6 @@ func Build(options configapi.NodeConfig) (*kubeletoptions.KubeletServer, *compon
}
}

proxyconfig, err := buildKubeProxyConfig(options)
if err != nil {
return nil, nil, err
}

if network.IsOpenShiftNetworkPlugin(options.NetworkConfig.NetworkPluginName) {
// SDN plugin pod setup/teardown is implemented as a CNI plugin
server.NetworkPluginName = kubeletcni.CNIPluginName
Expand All @@ -152,72 +146,7 @@ func Build(options configapi.NodeConfig) (*kubeletoptions.KubeletServer, *compon
server.HairpinMode = componentconfig.HairpinNone
}

return server, proxyconfig, nil
}

func buildKubeProxyConfig(options configapi.NodeConfig) (*componentconfig.KubeProxyConfiguration, error) {
proxyOptions, err := kubeproxyoptions.NewOptions()
if err != nil {
return nil, err
}
// get default config
proxyconfig := proxyOptions.GetConfig()

// BindAddress - Override default bind address from our config
addr := options.ServingInfo.BindAddress
host, _, err := net.SplitHostPort(addr)
if err != nil {
return nil, fmt.Errorf("The provided value to bind to must be an ip:port %q", addr)
}
ip := net.ParseIP(host)
if ip == nil {
return nil, fmt.Errorf("The provided value to bind to must be an ip:port: %q", addr)
}
proxyconfig.BindAddress = ip.String()
// MetricsBindAddress - disable
proxyconfig.MetricsBindAddress = ""

// OOMScoreAdj, ResourceContainer - clear, we don't run in a container
oomScoreAdj := int32(0)
proxyconfig.OOMScoreAdj = &oomScoreAdj
proxyconfig.ResourceContainer = ""

// use the same client as the node
proxyconfig.ClientConnection.KubeConfigFile = options.MasterKubeConfig

// ProxyMode, set to iptables
proxyconfig.Mode = "iptables"

// IptablesSyncPeriod, set to our config value
syncPeriod, err := time.ParseDuration(options.IPTablesSyncPeriod)
if err != nil {
return nil, fmt.Errorf("Cannot parse the provided ip-tables sync period (%s) : %v", options.IPTablesSyncPeriod, err)
}
proxyconfig.IPTables.SyncPeriod = metav1.Duration{
Duration: syncPeriod,
}
masqueradeBit := int32(0)
proxyconfig.IPTables.MasqueradeBit = &masqueradeBit

// PortRange, use default
// HostnameOverride, use default
// ConfigSyncPeriod, use default
// MasqueradeAll, use default
// CleanupAndExit, use default
// KubeAPIQPS, use default, doesn't apply until we build a separate client
// KubeAPIBurst, use default, doesn't apply until we build a separate client
// UDPIdleTimeout, use default

// Resolve cmd flags to add any user overrides
if err := cmdflags.Resolve(options.ProxyArguments, proxyOptions.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}

if err := proxyOptions.Complete(); err != nil {
return nil, err
}

return proxyconfig, nil
return server, nil
}

func ToFlags(config *kubeletoptions.KubeletServer) []string {
Expand Down
Loading

0 comments on commit d349f9b

Please sign in to comment.