Merge pull request #683 from JoshVanL/435-calico-kubernetes-backend

Feature flags for calico with CRDs and optionally typha

docs/spelling_wordlist.txt

@@ -103,6 +103,7 @@ testability
 Todo
 toolkit
 ttl
+typha
 ubuntu
 uid
 unsealer

docs/user-guide.rst

@@ -928,6 +928,35 @@ file under the Kubernetes code block like following:
        featureGates:
          CPUManager: false
 
+Calico Backend
+~~~~~~~~~~~~~~
+
+By default Tarmak will deploy Calico into your Kubernetes cluster, configured to
+use etcd as the backend. Calico also supports using the Kubernetes API server
+instead, which can be configured by changing the Calico option in the Tarmak
+config like the following:
+
+.. code-block:: yaml
+
+   kubernetes:
+    calico:
+      backend: kubernetes
+      enableTypha: true
+      typhaReplicas: 1
+
+This change will take effect cluster wide.
+
+Calico also supports using Typha, a purpose built, fan-out daemon to reduce load
+on the targeted data store. More information can be found on it's `project page
+<https://github.com/projectcalico/typha>`_.
+
+.. note::
+   Typha should only typically be enabled when your Kubernetes node count
+   exceeds 50.
+
+Enabling Typha, along with setting the number of replicas is shown above.
+
+
 Cluster Services
 ----------------
 

pkg/apis/cluster/v1alpha1/cluster.go

@@ -44,6 +44,11 @@ const (
 	PrometheusModeExternalScrapeTargetsOnly = "ExternalScrapeTargetsOnly"
 )
 
+const (
+	CalicoBackendEtcd       ClusterKubernetesCalicoBackend = "etcd"
+	CalicoBackendKubernetes ClusterKubernetesCalicoBackend = "kubernetes"
+)
+
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 // +k8s:openapi-gen=true
 // +resource:path=clusters
@@ -97,6 +102,7 @@ type ClusterKubernetes struct {
 	Scheduler         *ClusterKubernetesScheduler         `json:"scheduler,omitempty"`
 	Proxy             *ClusterKubernetesProxy             `json:"proxy,omitempty"`
 	ControllerManager *ClusterKubernetesControllerManager `json:"controllerManager,omitempty"`
+	Calico            *ClusterKubernetesCalico            `json:"calico,omitempty"`
 
 	GlobalFeatureGates map[string]bool `json:"globalFeatureGates,omitempty"`
 }
@@ -228,6 +234,14 @@ type ClusterKubernetesControllerManager struct {
 	FeatureGates map[string]bool `json:"featureGates,omitempty"`
 }
 
+type ClusterKubernetesCalicoBackend string
+type ClusterKubernetesCalico struct {
+	Backend ClusterKubernetesCalicoBackend `json:"backend"`
+
+	EnableTypha   bool `json:"enableTypha"`
+	TyphaReplicas *int `json:"typhaReplicas"`
+}
+
 // +k8s:openapi-gen=true
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 

pkg/apis/cluster/v1alpha1/defaults.go

@@ -71,6 +71,22 @@ func SetDefaults_Cluster(obj *Cluster) {
 		obj.Kubernetes.Dashboard = &ClusterKubernetesDashboard{}
 	}
 
+	if obj.Kubernetes.Calico == nil {
+		obj.Kubernetes.Calico = &ClusterKubernetesCalico{
+			Backend:     "etcd",
+			EnableTypha: false,
+		}
+	}
+
+	if obj.Kubernetes.Calico.Backend == "" {
+		obj.Kubernetes.Calico.Backend = "etcd"
+	}
+
+	if obj.Kubernetes.Calico.EnableTypha &&
+		obj.Kubernetes.Calico.TyphaReplicas == nil {
+		obj.Kubernetes.Calico.TyphaReplicas = intPointer(1)
+	}
+
 	// EBS encryption off if Amazon interface used
 	// but EBSEncrypted not specified
 	if obj.Amazon == nil {
@@ -121,6 +137,7 @@ func SetDefaults_Cluster(obj *Cluster) {
 	if obj.VaultHelper == nil {
 		obj.VaultHelper = new(ClusterVaultHelper)
 	}
+
 }
 
 func boolPointer(x bool) *bool {
@@ -131,6 +148,10 @@ func floatPointer(x float64) *float64 {
 	return &x
 }
 
+func intPointer(x int) *int {
+	return &x
+}
+
 func allocateAmazonESProxyPort(loggingSinks []*LoggingSink) int {
 
 	allocatedPorts := make(map[int]struct{})

pkg/puppet/puppet.go

@@ -228,6 +228,20 @@ func kubernetesClusterConfig(conf *clusterv1alpha1.ClusterKubernetes, hieraData
 		hieraData.variables = append(hieraData.variables, fmt.Sprintf(`kubernetes::controller_manager::feature_gates:%s`, gates))
 	}
 
+	if conf.Calico != nil {
+		hieraData.variables = append(hieraData.variables, fmt.Sprintf("calico::backend: %s", conf.Calico.Backend))
+
+		hieraData.variables = append(hieraData.variables, fmt.Sprintf("calico::typha_enabled: %v", conf.Calico.EnableTypha))
+		if conf.Calico.TyphaReplicas != nil {
+			hieraData.variables = append(hieraData.variables, fmt.Sprintf("calico::typha_replicas: %v", *conf.Calico.TyphaReplicas))
+		}
+
+		if conf.Calico.Backend == "kubernetes" {
+			hieraData.variables = append(hieraData.variables, "kubernetes::controller_manager::allocate_node_cidrs: true")
+			hieraData.variables = append(hieraData.variables, "calico::node::ipv4_pool_ipip_mode: cross-subnet")
+		}
+	}
+
 	return
 }
 

pkg/tarmak/cluster/cluster.go

@@ -444,6 +444,13 @@ func (c *Cluster) Validate() error {
 				result = multierror.Append(result, err)
 			}
 		}
+
+		//validate calico
+		if c.Config().Kubernetes.Calico != nil {
+			if err := c.validateCalico(); err != nil {
+				result = multierror.Append(result, err)
+			}
+		}
 	}
 
 	return result.ErrorOrNil()
@@ -594,6 +601,36 @@ func (c *Cluster) validatePrometheusMode() error {
 	return result
 }
 
+func (c *Cluster) validateCalico() error {
+	var result *multierror.Error
+
+	calico := c.Config().Kubernetes.Calico
+	if calico.Backend != clusterv1alpha1.CalicoBackendEtcd &&
+		calico.Backend != clusterv1alpha1.CalicoBackendKubernetes {
+		result = multierror.Append(result, fmt.Errorf(
+			"calico's backend may only be set to [%s %s], got=%s",
+			clusterv1alpha1.CalicoBackendEtcd, clusterv1alpha1.CalicoBackendKubernetes, calico.Backend))
+	}
+
+	if calico.Backend != clusterv1alpha1.CalicoBackendKubernetes && calico.EnableTypha {
+		result = multierror.Append(result, fmt.Errorf(
+			"typha enabled but backend is not 'kubernetes', got=%s", calico.Backend))
+	}
+
+	if calico.EnableTypha &&
+		(calico.TyphaReplicas == nil || *calico.TyphaReplicas <= 0) {
+		got := "nil"
+		if calico.TyphaReplicas != nil {
+			got = strconv.Itoa(*calico.TyphaReplicas)
+		}
+
+		result = multierror.Append(result, fmt.Errorf(
+			"typha enabled so expecting a non-zero positive replica count, got=%s", got))
+	}
+
+	return result.ErrorOrNil()
+}
+
 // Determine if this Cluster is a cluster or hub, single or multi environment
 func (c *Cluster) Type() string {
 	if c.conf.Type != "" {

puppet/modules/calico/manifests/config.pp

@@ -4,19 +4,31 @@
 
   $mtu = $::calico::mtu
   $namespace = $::calico::namespace
-  $etcd_endpoints = $::calico::etcd_endpoints
 
-  $etcd_proto = $::calico::etcd_proto
-  if $etcd_proto == 'https' {
-    $etcd_tls_dir = $::calico::etcd_tls_dir
-    $etcd_ca_file = $::calico::etcd_ca_file
-    $etcd_cert_file = $::calico::etcd_cert_file
-    $etcd_key_file = $::calico::etcd_key_file
-  }
+  if $::calico::backend == 'etcd' {
+    $etcd_endpoints = $::calico::etcd_endpoints
+    $etcd_proto = $::calico::etcd_proto
+
+    if $etcd_proto == 'https' {
+      $etcd_tls_dir = $::calico::etcd_tls_dir
+      $etcd_ca_file = $::calico::etcd_ca_file
+      $etcd_cert_file = $::calico::etcd_cert_file
+      $etcd_key_file = $::calico::etcd_key_file
+    }
+
+    kubernetes::apply{'calico-config':
+      manifests => [
+        template('calico/configmap_etcd.yaml.erb'),
+      ],
+    }
+
+  } else {
+    $pod_network = $::calico::pod_network
 
-  kubernetes::apply{'calico-config':
-    manifests => [
-      template('calico/configmap.yaml.erb'),
-    ],
+    kubernetes::apply{'calico-config':
+      manifests => [
+        template('calico/configmap_kubernetes.yaml.erb'),
+      ],
+    }
   }
 }

puppet/modules/calico/manifests/init.pp

@@ -8,6 +8,8 @@
   String $etcd_key_file = '',
   String $cloud_provider = $::calico::params::cloud_provider,
   String $namespace = 'kube-system',
+  Boolean $typha_enabled = false,
+  Optional[Integer] $typha_replicas = undef,
   Optional[String] $pod_network = undef,
   Integer[1000,65535] $mtu = 1480,
 ) inherits ::calico::params
@@ -29,8 +31,6 @@
       $etcd_proto = 'http'
     }
     $etcd_endpoints = $etcd_cluster.map |$node| { "${etcd_proto}://${node}:${etcd_overlay_port}" }.join(',')
-  } elsif $backend == 'kubernetes' {
-    fail('Backend storage kubernetes is not yet supported')
   }
 
   if $cloud_provider == 'aws' {

puppet/modules/calico/manifests/node.pp

@@ -4,9 +4,7 @@
 # workern node.
 # @param metrics_port Port for felix metrics endpoint, 0 disables metrics collection
 class calico::node (
-  String $node_image = 'quay.io/calico/node',
   String $node_version = '3.1.4',
-  String $cni_image = 'quay.io/calico/cni',
   String $cni_version = '3.1.4',
   Enum['always', 'cross-subnet', 'off'] $ipv4_pool_ipip_mode = 'always',
   Integer[0,65535] $metrics_port = 9091,
@@ -18,8 +16,11 @@
   $namespace = $::calico::namespace
   $mtu = $::calico::mtu
   $ipv4_pool_cidr = $::calico::pod_network
-  $etcd_cert_path = $::calico::etcd_cert_path
-  $etcd_proto = $::calico::etcd_proto
+  $backend = $::calico::backend
+
+  $typha_enabled  = $::calico::typha_enabled
+  $typha_replicas = $::calico::typha_replicas
+
 
   $authorization_mode = $::kubernetes::_authorization_mode
   if member($authorization_mode, 'RBAC'){
@@ -34,10 +35,35 @@
     $version_before_1_6 = true
   }
 
+  if $backend == 'etcd' {
+    $etcd_cert_path = $::calico::etcd_cert_path
+    $etcd_proto = $::calico::etcd_proto
+    $node_image = 'quay.io/calico/node'
+    $cni_image = 'quay.io/calico/cni'
+
+    $manifests = ''
+
+  } else {
+    $node_image = 'calico/node'
+    $cni_image = 'calico/cni'
+
+    if $typha_enabled {
+      $manifests = [
+          template('calico/node-crd.yaml.erb'),
+          template('calico/node-typha.yaml.erb'),
+        ]
+    } else {
+      $manifests = [
+          template('calico/node-crd.yaml.erb'),
+        ]
+    }
+  }
+
   kubernetes::apply{'calico-node':
     manifests => [
-      template('calico/node-daemonset.yaml.erb'),
       template('calico/node-rbac.yaml.erb'),
+      template('calico/node-daemonset.yaml.erb'),
+      $manifests,
     ],
   }