Allowing multiple CIDR addresses for allocation of Nodes

Chaning the Network Config section of the the master config to
allow multiple CIDR addresses and hostsubnet Lengths for the
allocation of nodes' address space

pkg/cmd/server/api/serialization_test.go

@@ -112,13 +112,33 @@ func fuzzInternalObject(t *testing.T, forVersion schema.GroupVersion, item runti
 					obj.NetworkConfig.ServiceNetworkCIDR = "10.0.0.0/24"
 				}
 			}
+			if len(obj.NetworkConfig.ClusterNetworks) == 0 {
+				clusterNetwork := []configapi.ClusterNetworkEntry{
+					{
+						CIDR:             "10.128.0.0/14",
+						HostSubnetLength: 9,
+					},
+				}
+				obj.NetworkConfig.ClusterNetworks = clusterNetwork
+			}
 
 			// TODO stop duplicating the conversion in the test.
 			kubeConfig := obj.KubernetesMasterConfig
 			noCloudProvider := kubeConfig != nil && (len(kubeConfig.ControllerArguments["cloud-provider"]) == 0 || kubeConfig.ControllerArguments["cloud-provider"][0] == "")
 			if noCloudProvider && len(obj.NetworkConfig.IngressIPNetworkCIDR) == 0 {
 				cidr := configapi.DefaultIngressIPNetworkCIDR
-				if !(configapi.CIDRsOverlap(cidr, obj.NetworkConfig.ClusterNetworkCIDR) || configapi.CIDRsOverlap(cidr, obj.NetworkConfig.ServiceNetworkCIDR)) {
+				setCIDR := true
+				if configapi.CIDRsOverlap(cidr, obj.NetworkConfig.ServiceNetworkCIDR) {
+					setCIDR = false
+				} else {
+					for _, clusterNetwork := range obj.NetworkConfig.ClusterNetworks {
+						if configapi.CIDRsOverlap(cidr, clusterNetwork.CIDR) {
+							setCIDR = false
+							break
+						}
+					}
+				}
+				if setCIDR {
 					obj.NetworkConfig.IngressIPNetworkCIDR = cidr
 				}
 			}

pkg/cmd/server/api/types.go

@@ -667,10 +667,12 @@ type UserAgentDenyRule struct {
 
 // MasterNetworkConfig to be passed to the compiled in network plugin
 type MasterNetworkConfig struct {
-	NetworkPluginName  string
-	ClusterNetworkCIDR string
-	HostSubnetLength   uint32
-	ServiceNetworkCIDR string
+	NetworkPluginName            string
+	DeprecatedClusterNetworkCIDR string
+	// ClusterNetworks contains a list of cluster networks that defines the global overlay networks L3 space.
+	ClusterNetworks            []ClusterNetworkEntry
+	DeprecatedHostSubnetLength uint32
+	ServiceNetworkCIDR         string
 	// ExternalIPNetworkCIDRs controls what values are acceptable for the service external IP field. If empty, no externalIP
 	// may be set. It may contain a list of CIDRs which are checked for access. If a CIDR is prefixed with !, IPs in that
 	// CIDR will be rejected. Rejections will be applied first, then the IP checked against one of the allowed CIDRs. You
@@ -683,6 +685,15 @@ type MasterNetworkConfig struct {
 	IngressIPNetworkCIDR string
 }
 
+// ClusterNetworkEntry defines an individual cluster network. The CIDRs cannot overlap with other cluster network CIDRs, CIDRs
+// reserved for external ips, CIDRs reserved for service networks, and CIDRs reserved for ingress ips.
+type ClusterNetworkEntry struct {
+	// CIDR defines the total range of a cluster networks address space.
+	CIDR string
+	// HostSubnetLength gives the number of address bits reserved for pod IPs on each node.
+	HostSubnetLength uint32
+}
+
 type ImageConfig struct {
 	// Format describes how to determine image names for system components
 	Format string

pkg/cmd/server/api/v1/conversions.go

@@ -90,14 +90,31 @@ func SetDefaults_MasterConfig(obj *MasterConfig) {
 			obj.NetworkConfig.ServiceNetworkCIDR = "10.0.0.0/24"
 		}
 	}
+	if len(obj.NetworkConfig.ClusterNetworks) == 0 {
+		obj.NetworkConfig.ClusterNetworks = []ClusterNetworkEntry{{CIDR: obj.NetworkConfig.DeprecatedClusterNetworkCIDR, HostSubnetLength: obj.NetworkConfig.DeprecatedHostSubnetLength}}
+
+		obj.NetworkConfig.DeprecatedClusterNetworkCIDR = ""
+		obj.NetworkConfig.DeprecatedHostSubnetLength = 0
+	}
 
 	// TODO Detect cloud provider when not using built-in kubernetes
 	kubeConfig := obj.KubernetesMasterConfig
 	noCloudProvider := kubeConfig != nil && (len(kubeConfig.ControllerArguments["cloud-provider"]) == 0 || kubeConfig.ControllerArguments["cloud-provider"][0] == "")
 
 	if noCloudProvider && len(obj.NetworkConfig.IngressIPNetworkCIDR) == 0 {
 		cidr := internal.DefaultIngressIPNetworkCIDR
-		if !(internal.CIDRsOverlap(cidr, obj.NetworkConfig.ClusterNetworkCIDR) || internal.CIDRsOverlap(cidr, obj.NetworkConfig.ServiceNetworkCIDR)) {
+		cidrOverlap := false
+		if internal.CIDRsOverlap(cidr, obj.NetworkConfig.ServiceNetworkCIDR) {
+			cidrOverlap = true
+		} else {
+			for _, entry := range obj.NetworkConfig.ClusterNetworks {
+				if internal.CIDRsOverlap(cidr, entry.CIDR) {
+					cidrOverlap = true
+					break
+				}
+			}
+		}
+		if !cidrOverlap {
 			obj.NetworkConfig.IngressIPNetworkCIDR = cidr
 		}
 	}

pkg/cmd/server/api/v1/types.go

@@ -543,10 +543,12 @@ type RoutingConfig struct {
 type MasterNetworkConfig struct {
 	// NetworkPluginName is the name of the network plugin to use
 	NetworkPluginName string `json:"networkPluginName"`
-	// ClusterNetworkCIDR is the CIDR string to specify the global overlay network's L3 space
-	ClusterNetworkCIDR string `json:"clusterNetworkCIDR"`
-	// HostSubnetLength is the number of bits to allocate to each host's subnet e.g. 8 would mean a /24 network on the host
-	HostSubnetLength uint32 `json:"hostSubnetLength"`
+	// DeprecatedClusterNetworkCIDR is the CIDR string to specify the global overlay network's L3 space.  Deprecated, but maintained for backwards compatibility, use ClusterNetworks instead.
+	DeprecatedClusterNetworkCIDR string `json:"clusterNetworkCIDR,omitempty"`
+	// ClusterNetworks is a list of ClusterNetwork objects that defines the global overlay network's L3 space by specifying a set of CIDR and netmasks that the SDN can allocate addressed from.  If this is specified, then DeprecatedClusterNetworkCIDR and DeprecatedHostSubnetLength may not be set.
+	ClusterNetworks []ClusterNetworkEntry `json:"clusterNetworks"`
+	// DeprecatedHostSubnetLength is the number of bits to allocate to each host's subnet e.g. 8 would mean a /24 network on the host.  Deprecated, but maintained for backwards compatibility, use ClusterNetworks instead.
+	DeprecatedHostSubnetLength uint32 `json:"hostSubnetLength,omitempty"`
 	// ServiceNetwork is the CIDR string to specify the service networks
 	ServiceNetworkCIDR string `json:"serviceNetworkCIDR"`
 	// ExternalIPNetworkCIDRs controls what values are acceptable for the service external IP field. If empty, no externalIP
@@ -561,6 +563,14 @@ type MasterNetworkConfig struct {
 	IngressIPNetworkCIDR string `json:"ingressIPNetworkCIDR"`
 }
 
+// ClusterNetworkEntry defines an individual cluster network. The CIDRs cannot overlap with other cluster network CIDRs, CIDRs reserved for external ips, CIDRs reserved for service networks, and CIDRs reserved for ingress ips.
+type ClusterNetworkEntry struct {
+	// CIDR defines the total range of a cluster networks address space.
+	CIDR string `json:"cidr"`
+	// HostSubnetLength is the number of bits of the accompanying CIDR address to allocate to each node. eg, 8 would mean that each node would have a /24 slice of the overlay network for its pod.
+	HostSubnetLength uint32 `json:"hostSubnetLength"`
+}
+
 // ImageConfig holds the necessary configuration options for building image names for system components
 type ImageConfig struct {
 	// Format is the format of the name to be built for the system component

pkg/cmd/server/api/v1/types_test.go

@@ -220,9 +220,8 @@ masterClients:
   openshiftLoopbackKubeConfig: ""
 masterPublicURL: ""
 networkConfig:
-  clusterNetworkCIDR: ""
+  clusterNetworks: null
   externalIPNetworkCIDRs: null
-  hostSubnetLength: 0
   ingressIPNetworkCIDR: ""
   networkPluginName: ""
   serviceNetworkCIDR: ""

pkg/cmd/server/api/validation/master.go

@@ -165,6 +165,7 @@ func ValidateMasterConfig(config *api.MasterConfig, fldPath *field.Path) Validat
 	}
 
 	validationResults.AddErrors(ValidateIngressIPNetworkCIDR(config, fldPath.Child("networkConfig", "ingressIPNetworkCIDR").Index(0))...)
+	validationResults.Append(ValidateDeprecatedClusterNetworkConfig(config, fldPath.Child("networkConfig")))
 
 	validationResults.AddErrors(ValidateKubeConfig(config.MasterClients.OpenShiftLoopbackKubeConfig, fldPath.Child("masterClients", "openShiftLoopbackKubeConfig"))...)
 
@@ -830,8 +831,10 @@ func ValidateIngressIPNetworkCIDR(config *api.MasterConfig, fldPath *field.Path)
 	noCloudProvider := kubeConfig != nil && (len(kubeConfig.ControllerArguments["cloud-provider"]) == 0 || kubeConfig.ControllerArguments["cloud-provider"][0] == "")
 
 	if noCloudProvider {
-		if api.CIDRsOverlap(cidr, config.NetworkConfig.ClusterNetworkCIDR) {
-			addError("conflicts with cluster network CIDR")
+		for _, entry := range config.NetworkConfig.ClusterNetworks {
+			if api.CIDRsOverlap(cidr, entry.CIDR) {
+				addError(fmt.Sprintf("conflicts with cluster network CIDR: %s", entry.CIDR))
+			}
 		}
 		if api.CIDRsOverlap(cidr, config.NetworkConfig.ServiceNetworkCIDR) {
 			addError("conflicts with service network CIDR")
@@ -842,3 +845,15 @@ func ValidateIngressIPNetworkCIDR(config *api.MasterConfig, fldPath *field.Path)
 
 	return
 }
+
+func ValidateDeprecatedClusterNetworkConfig(config *api.MasterConfig, fldPath *field.Path) ValidationResults {
+	validationResults := ValidationResults{}
+
+	if len(config.NetworkConfig.ClusterNetworks) > 0 && config.NetworkConfig.DeprecatedHostSubnetLength != 0 {
+		validationResults.AddErrors(field.Invalid(fldPath.Child("hostSubnetLength"), config.NetworkConfig.DeprecatedHostSubnetLength, "cannot set hostSubnetLength and clusterNetworks, please use clusterNetworks"))
+	}
+	if len(config.NetworkConfig.ClusterNetworks) > 0 && config.NetworkConfig.DeprecatedClusterNetworkCIDR != "" {
+		validationResults.AddErrors(field.Invalid(fldPath.Child("clusterNetworkCIDR"), config.NetworkConfig.DeprecatedClusterNetworkCIDR, "cannot set clusterNetworkCIDR and clusterNetworks, please use clusterNetworks"))
+	}
+	return validationResults
+}

pkg/cmd/server/api/validation/master_test.go

@@ -495,7 +495,11 @@ func TestValidateIngressIPNetworkCIDR(t *testing.T) {
 			NetworkConfig: configapi.MasterNetworkConfig{
 				IngressIPNetworkCIDR: test.cidr,
 				ServiceNetworkCIDR:   test.serviceCIDR,
-				ClusterNetworkCIDR:   test.clusterCIDR,
+				ClusterNetworks: []configapi.ClusterNetworkEntry{
+					{
+						CIDR: test.clusterCIDR,
+					},
+				},
 			},
 		}
 		errors := ValidateIngressIPNetworkCIDR(config, nil)

pkg/cmd/server/origin/master_config.go

@@ -654,7 +654,12 @@ func newAdmissionChain(pluginNames []string, admissionConfigFilename string, plu
 
 		case serviceadmit.RestrictedEndpointsPluginName:
 			// we need to set some customer parameters, so create by hand
-			restrictedNetworks, err := serviceadmit.ParseSimpleCIDRRules([]string{options.NetworkConfig.ClusterNetworkCIDR, options.NetworkConfig.ServiceNetworkCIDR})
+			var restricted []string
+			restricted = append(restricted, options.NetworkConfig.ServiceNetworkCIDR)
+			for _, cidr := range options.NetworkConfig.ClusterNetworks {
+				restricted = append(restricted, cidr.CIDR)
+			}
+			restrictedNetworks, err := serviceadmit.ParseSimpleCIDRRules(restricted)
 			if err != nil {
 				// should have been caught with validation
 				return nil, err

pkg/cmd/server/start/master_args.go

@@ -294,9 +294,13 @@ func (args MasterArgs) BuildSerializeableMasterConfig() (*configapi.MasterConfig
 		},
 
 		NetworkConfig: configapi.MasterNetworkConfig{
-			NetworkPluginName:  args.NetworkArgs.NetworkPluginName,
-			ClusterNetworkCIDR: args.NetworkArgs.ClusterNetworkCIDR,
-			HostSubnetLength:   args.NetworkArgs.HostSubnetLength,
+			NetworkPluginName: args.NetworkArgs.NetworkPluginName,
+			ClusterNetworks: []configapi.ClusterNetworkEntry{
+				{
+					CIDR:             args.NetworkArgs.ClusterNetworkCIDR,
+					HostSubnetLength: args.NetworkArgs.HostSubnetLength,
+				},
+			},
 			ServiceNetworkCIDR: args.NetworkArgs.ServiceNetworkCIDR,
 		},
 

pkg/network/apis/network/types.go

@@ -17,12 +17,18 @@ type ClusterNetwork struct {
 	metav1.TypeMeta
 	metav1.ObjectMeta
 
+	ClusterNetworks  []ClusterNetworkEntry
 	Network          string
 	HostSubnetLength uint32
 	ServiceNetwork   string
 	PluginName       string
 }
 
+type ClusterNetworkEntry struct {
+	CIDR             string
+	HostSubnetLength uint32
+}
+
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
 type ClusterNetworkList struct {

pkg/network/apis/network/v1/types.go

@@ -21,17 +21,27 @@ type ClusterNetwork struct {
 	metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
 
 	// Network is a CIDR string specifying the global overlay network's L3 space
-	Network string `json:"network" protobuf:"bytes,2,opt,name=network"`
+	Network string `json:"network,omitempty" protobuf:"bytes,2,opt,name=network"`
 	// HostSubnetLength is the number of bits of network to allocate to each node. eg, 8 would mean that each node would have a /24 slice of the overlay network for its pods
-	HostSubnetLength uint32 `json:"hostsubnetlength" protobuf:"varint,3,opt,name=hostsubnetlength"`
+	HostSubnetLength uint32 `json:"hostsubnetlength,omitempty" protobuf:"varint,3,opt,name=hostsubnetlength"`
 	// ServiceNetwork is the CIDR range that Service IP addresses are allocated from
 	ServiceNetwork string `json:"serviceNetwork" protobuf:"bytes,4,opt,name=serviceNetwork"`
 	// PluginName is the name of the network plugin being used
 	PluginName string `json:"pluginName,omitempty" protobuf:"bytes,5,opt,name=pluginName"`
+	// ClusterNetworks is a list of ClusterNetwork objects that defines the global overlay network's L3 space by specifying a set of CIDR and netmasks that the SDN can allocate addressed from.  If this is specified, then DeprecatedClusterNetworkCIDR and DeprecatedHostSubnetLength may not be set.
+	ClusterNetworks []ClusterNetworkEntry `json:"clusterNetworks" protobuf:"bytes,6,rep,name=clusterNetworks"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
+// ClusterNetworkEntry defines an individual cluster network. The CIDRs cannot overlap with other cluster network CIDRs, CIDRs reserved for external ips, CIDRs reserved for service networks, and CIDRs reserved for ingress ips.
+type ClusterNetworkEntry struct {
+	// CIDR defines the total range of a cluster networks address space.
+	CIDR string `json:"CIDR" protobuf:"bytes,1,opt,name=cidr"`
+	// HostSubnetLength is the number of bits of the accompanying CIDR address to allocate to each node. eg, 8 would mean that each node would have a /24 slice of the overlay network for its pods.
+	HostSubnetLength uint32 `json:"hostSubnetLength" protobuf:"varint,2,opt,name=hostSubnetLength"`
+}
+
 // ClusterNetworkList is a collection of ClusterNetworks
 type ClusterNetworkList struct {
 	metav1.TypeMeta `json:",inline"`

pkg/network/apis/network/validation/validation.go

@@ -3,12 +3,14 @@ package validation
 import (
 	"fmt"
 	"net"
+	"reflect"
 
 	"k8s.io/apimachinery/pkg/api/validation/path"
 	utilvalidation "k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/kubernetes/pkg/api/validation"
 
+	configapi "github.com/openshift/origin/pkg/cmd/server/api"
 	"github.com/openshift/origin/pkg/network"
 	networkapi "github.com/openshift/origin/pkg/network/apis/network"
 	"github.com/openshift/origin/pkg/util/netutils"
@@ -24,29 +26,53 @@ func SetDefaultClusterNetwork(cn networkapi.ClusterNetwork) {
 // ValidateClusterNetwork tests if required fields in the ClusterNetwork are set, and ensures that the "default" ClusterNetwork can only be set to the correct values
 func ValidateClusterNetwork(clusterNet *networkapi.ClusterNetwork) field.ErrorList {
 	allErrs := validation.ValidateObjectMeta(&clusterNet.ObjectMeta, false, path.ValidatePathSegmentName, field.NewPath("metadata"))
+	var testedCIDRS []*net.IPNet
 
-	clusterIPNet, err := netutils.ParseCIDRMask(clusterNet.Network)
-	if err != nil {
-		allErrs = append(allErrs, field.Invalid(field.NewPath("network"), clusterNet.Network, err.Error()))
-	} else {
-		maskLen, addrLen := clusterIPNet.Mask.Size()
-		if clusterNet.HostSubnetLength > uint32(addrLen-maskLen) {
-			allErrs = append(allErrs, field.Invalid(field.NewPath("hostSubnetLength"), clusterNet.HostSubnetLength, "subnet length is too large for clusterNetwork"))
-		} else if clusterNet.HostSubnetLength < 2 {
-			allErrs = append(allErrs, field.Invalid(field.NewPath("hostSubnetLength"), clusterNet.HostSubnetLength, "subnet length must be at least 2"))
+	if len(clusterNet.Network) != 0 || clusterNet.HostSubnetLength != 0 {
+		//In the case that a user manually makes a clusterNetwork object with clusterNet.Network and clusterNet.HostubnetLength at least make sure they are valid values
+		clusterIPNet, err := netutils.ParseCIDRMask(clusterNet.Network)
+		if err != nil {
+			allErrs = append(allErrs, field.Invalid(field.NewPath("network"), clusterNet.Network, err.Error()))
+		} else {
+			maskLen, addrLen := clusterIPNet.Mask.Size()
+			if clusterNet.HostSubnetLength > uint32(addrLen-maskLen) {
+				allErrs = append(allErrs, field.Invalid(field.NewPath("hostSubnetLength"), clusterNet.HostSubnetLength, "subnet length is too large for clusterNetwork"))
+			} else if clusterNet.HostSubnetLength < 2 {
+				allErrs = append(allErrs, field.Invalid(field.NewPath("hostSubnetLength"), clusterNet.HostSubnetLength, "subnet length must be at least 2"))
+			}
 		}
 	}
 
+	if len(clusterNet.ClusterNetworks) == 0 && len(clusterNet.Network) == 0 {
+		allErrs = append(allErrs, field.Invalid(field.NewPath("clusterNetworks"), clusterNet.ClusterNetworks, "must have at least one cluster network CIDR"))
+	}
 	serviceIPNet, err := netutils.ParseCIDRMask(clusterNet.ServiceNetwork)
 	if err != nil {
 		allErrs = append(allErrs, field.Invalid(field.NewPath("serviceNetwork"), clusterNet.ServiceNetwork, err.Error()))
 	}
+	for i, cn := range clusterNet.ClusterNetworks {
+		clusterIPNet, err := netutils.ParseCIDRMask(cn.CIDR)
+		if err != nil {
+			allErrs = append(allErrs, field.Invalid(field.NewPath("clusterNetworks").Index(i).Child("cidr"), cn.CIDR, err.Error()))
+			continue
+		}
+		maskLen, addrLen := clusterIPNet.Mask.Size()
+		if cn.HostSubnetLength > uint32(addrLen-maskLen) {
+			allErrs = append(allErrs, field.Invalid(field.NewPath("clusterNetworks").Index(i).Child("hostSubnetLength"), cn.HostSubnetLength, "subnet length is too large for clusterNetwork "))
+		} else if cn.HostSubnetLength < 2 {
+			allErrs = append(allErrs, field.Invalid(field.NewPath("clusterNetworks").Index(i).Child("hostSubnetLength"), cn.HostSubnetLength, "subnet length must be at least 2"))
+		}
 
-	if (clusterIPNet != nil) && (serviceIPNet != nil) && clusterIPNet.Contains(serviceIPNet.IP) {
-		allErrs = append(allErrs, field.Invalid(field.NewPath("serviceNetwork"), clusterNet.ServiceNetwork, "service network overlaps with cluster network"))
-	}
-	if (serviceIPNet != nil) && (clusterIPNet != nil) && serviceIPNet.Contains(clusterIPNet.IP) {
-		allErrs = append(allErrs, field.Invalid(field.NewPath("network"), clusterNet.Network, "cluster network overlaps with service network"))
+		for _, cidr := range testedCIDRS {
+			if configapi.CIDRsOverlap(clusterIPNet.String(), cidr.String()) {
+				allErrs = append(allErrs, field.Invalid(field.NewPath("clusterNetworks").Index(i).Child("cidr"), cn.CIDR, fmt.Sprintf("cidr range overlaps with another cidr %q", cidr.String())))
+			}
+		}
+		testedCIDRS = append(testedCIDRS, clusterIPNet)
+
+		if (clusterIPNet != nil) && (serviceIPNet != nil) && configapi.CIDRsOverlap(clusterIPNet.String(), serviceIPNet.String()) {
+			allErrs = append(allErrs, field.Invalid(field.NewPath("serviceNetwork"), clusterNet.ServiceNetwork, fmt.Sprintf("service network overlaps with cluster network cidr: %s", clusterIPNet.String())))
+		}
 	}
 
 	if clusterNet.Name == networkapi.ClusterNetworkDefault && defaultClusterNetwork != nil {
@@ -56,6 +82,9 @@ func ValidateClusterNetwork(clusterNet *networkapi.ClusterNetwork) field.ErrorLi
 		if clusterNet.HostSubnetLength != defaultClusterNetwork.HostSubnetLength {
 			allErrs = append(allErrs, field.Invalid(field.NewPath("hostSubnetLength"), clusterNet.HostSubnetLength, "cannot change the default ClusterNetwork record via API."))
 		}
+		if !reflect.DeepEqual(clusterNet.ClusterNetworks, defaultClusterNetwork.ClusterNetworks) {
+			allErrs = append(allErrs, field.Invalid(field.NewPath("ClusterNetworks"), clusterNet.ClusterNetworks, "cannot change the default ClusterNetwork record via API"))
+		}
 		if clusterNet.ServiceNetwork != defaultClusterNetwork.ServiceNetwork {
 			allErrs = append(allErrs, field.Invalid(field.NewPath("serviceNetwork"), clusterNet.ServiceNetwork, "cannot change the default ClusterNetwork record via API."))
 		}