Add Support for Tolerations

.changelog/426.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+Add support for tolerations to Consul API Gateway Controller and GatewayClassConfig.
+```

config/crd/bases/api-gateway.consul.hashicorp.com_gatewayclassconfigs.yaml

@@ -160,6 +160,47 @@ spec:
                 - NodePort
                 - LoadBalancer
                 type: string
+              tolerations:
+                description: 'Tolerations allow the scheduler to schedule nodes with
+                  matching taints More Info: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/'
+                items:
+                  description: The pod this Toleration is attached to tolerates any
+                    taint that matches the triple <key,value,effect> using the matching
+                    operator <operator>.
+                  properties:
+                    effect:
+                      description: Effect indicates the taint effect to match. Empty
+                        means match all taint effects. When specified, allowed values
+                        are NoSchedule, PreferNoSchedule and NoExecute.
+                      type: string
+                    key:
+                      description: Key is the taint key that the toleration applies
+                        to. Empty means match all taint keys. If the key is empty,
+                        operator must be Exists; this combination means to match all
+                        values and all keys.
+                      type: string
+                    operator:
+                      description: Operator represents a key's relationship to the
+                        value. Valid operators are Exists and Equal. Defaults to Equal.
+                        Exists is equivalent to wildcard for value, so that a pod
+                        can tolerate all taints of a particular category.
+                      type: string
+                    tolerationSeconds:
+                      description: TolerationSeconds represents the period of time
+                        the toleration (which must be of effect NoExecute, otherwise
+                        this field is ignored) tolerates the taint. By default, it
+                        is not set, which means tolerate the taint forever (do not
+                        evict). Zero and negative values will be treated as 0 (evict
+                        immediately) by the system.
+                      format: int64
+                      type: integer
+                    value:
+                      description: Value is the taint value the toleration matches
+                        to. If the operator is Exists, the value should be empty,
+                        otherwise just a regular string.
+                      type: string
+                  type: object
+                type: array
               useHostPorts:
                 description: If this is set, then the Envoy container ports are mapped
                   to host ports.

internal/k8s/builder/gateway.go

@@ -199,6 +199,7 @@ func (b *GatewayDeploymentBuilder) podSpec() corev1.PodSpec {
 			},
 		},
 		NodeSelector:       b.gwConfig.Spec.NodeSelector,
+		Tolerations:        b.gwConfig.Spec.Tolerations,
 		ServiceAccountName: orDefault(b.gwConfig.Spec.ConsulSpec.AuthSpec.Account, defaultServiceAccount),
 		// the init container copies the binary into the
 		// next envoy container so we can decouple the envoy

internal/k8s/builder/testdata/static-mapping.deployment.golden.yaml

@@ -115,6 +115,11 @@ spec:
       nodeSelector:
         ingress-ready: "true"
       serviceAccountName: consul-api-gateway
+      tolerations:
+      - effect: NoSchedule
+        key: key1
+        operator: Equal
+        value: value1
       volumes:
       - emptyDir: {}
         name: bootstrap

internal/k8s/builder/testdata/static-mapping.yaml

@@ -18,6 +18,11 @@ spec:
     envoy: "envoy:1"
   nodeSelector:
     "ingress-ready": "true"
+  tolerations:
+    - key: "key1"
+      operator: "Equal"
+      value: "value1"
+      effect: "NoSchedule"
 ---
 apiVersion: gateway.networking.k8s.io/v1alpha2
 kind: GatewayClass

internal/testing/e2e/kind.go

@@ -5,25 +5,24 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/hashicorp/consul/sdk/freeport"
+	"github.com/vladimirvivien/gexe"
 	"html/template"
 	"io"
 	"io/ioutil"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"log"
 	"os"
 	"os/exec"
-	"strings"
-	"time"
-
-	"github.com/hashicorp/consul/sdk/freeport"
-	"github.com/vladimirvivien/gexe"
-	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/e2e-framework/klient"
 	"sigs.k8s.io/e2e-framework/klient/k8s/resources"
 	"sigs.k8s.io/e2e-framework/klient/wait"
 	"sigs.k8s.io/e2e-framework/klient/wait/conditions"
 	"sigs.k8s.io/e2e-framework/pkg/env"
 	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"strings"
+	"time"
 )
 
 var (
@@ -218,7 +217,6 @@ func (k *kindCluster) Destroy() error {
 	if err := os.RemoveAll(k.config); err != nil {
 		return fmt.Errorf("kind: remove config failed: %w", err)
 	}
-
 	return nil
 }
 

pkg/apis/v1alpha1/types.go

@@ -39,6 +39,9 @@ type GatewayClassConfigSpec struct {
 	// Selector which must match a node's labels for the pod to be scheduled on that node.
 	// More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
 	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
+	// Tolerations allow the scheduler to schedule nodes with matching taints
+	// More Info: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/
+	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
 	// If this is set, then the Envoy container ports are mapped
 	// to host ports.
 	UseHostPorts bool `json:"useHostPorts,omitempty"`