Support deploying CiliumL2AnnouncementPolicies

class/cilium.yml

@@ -25,6 +25,7 @@ parameters:
         - input_paths:
             - ${_base_directory}/component/aggregated-clusterroles.jsonnet
             - ${_base_directory}/component/egress-gateway-policies.jsonnet
+            - ${_base_directory}/component/l2-announcement-policies.jsonnet
             - ${_base_directory}/component/bgp-control-plane.jsonnet
             - ${_base_directory}/component/ocp-manage-kube-proxy.jsonnet
           input_type: jsonnet
@@ -51,6 +52,7 @@ parameters:
         - input_paths:
             - ${_base_directory}/component/aggregated-clusterroles.jsonnet
             - ${_base_directory}/component/egress-gateway-policies.jsonnet
+            - ${_base_directory}/component/l2-announcement-policies.jsonnet
             - ${_base_directory}/component/bgp-control-plane.jsonnet
             - ${_base_directory}/component/ocp-manage-kube-proxy.jsonnet
           input_type: jsonnet

class/defaults.yml

@@ -55,6 +55,11 @@ parameters:
         enabled: ${cilium:egress_gateway:enabled}
       bpf:
         masquerade: true
+      l2announcements:
+        enabled: ${cilium:l2_announcements:enabled}
+      k8sClientRateLimit:
+        qps: 15
+        burst: 30
       l7Proxy: ${cilium:_egressgw_l7proxy:${cilium:egress_gateway:enabled}}
       prometheus:
         enabled: true
@@ -87,6 +92,10 @@ parameters:
       generate_shadow_ranges_configmap: false
       egress_ip_ranges: {}
 
+    l2_announcements:
+      enabled: false
+      policies: {}
+
     bgp:
       enabled: false
       peerings: {}

component/l2-announcement-policies.jsonnet

@@ -0,0 +1,25 @@
+local com = import 'lib/commodore.libjsonnet';
+local kap = import 'lib/kapitan.libjsonnet';
+local kube = import 'lib/kube.libjsonnet';
+
+local inv = kap.inventory();
+local params = inv.parameters.cilium;
+
+local CiliumL2AnnouncementPolicy(name) =
+  kube._Object('cilium.io/v2alpha1', 'CiliumL2AnnouncementPolicy', name) {
+    metadata+: {
+      annotations+: {
+        'argocd.argoproj.io/sync-options': 'SkipDryRunOnMissingResource=true,Prune=false',
+      },
+    },
+  };
+
+local policies = com.generateResources(
+  params.l2_announcements.policies,
+  CiliumL2AnnouncementPolicy
+);
+
+{
+  [if params.l2_announcements.enabled && std.length(params.l2_announcements.policies) > 0 then
+    '40_l2_announcement_policies']: policies,
+}

docs/modules/ROOT/pages/references/parameters.adoc

@@ -542,6 +542,87 @@ spec:
 <2> The DaemonSet mounts the `eip-shadow-ranges` ConfigMap as a volume.
 <3> The DaemonSet is scheduled using the same node selector that's used for the `IsovalentEgressGatewayPolicy` resources
 
+
+== `l2_announcements`
+
+This section allows users to configure the [Cilium L2 Announcements / L2 Aware LB] feature.
+
+[NOTE]
+====
+The current implementation (and therefore examples shown here) has only been tested with Cilium EE.
+Please refer to the https://docs.cilium.io/en/stable/network/egress-gateway/#example-policy[example policy in the upstream documentation] for Cilium OSS.
+====
+
+=== `l2_announcements.enabled`
+
+[horizontal]
+type:: boolean
+default:: `false`
+
+This parameter allows users to set all the configurations necessary to enable the l2 announcement policy feature.
+
+[NOTE]
+====
+It is important to adjust the client rate limit when using this feature, due to increased API usage.
+See https://docs.cilium.io/en/latest/network/l2-announcements/#sizing-client-rate-limit[Sizing client rate limit] for sizing guidelines.
+====
+
+[NOTE]
+====
+Kube Proxy replacement mode must be enabled.
+====
+
+==== Example
+
+[source,yaml]
+----
+l2_announcements:
+  enabled: true
+cilium_helm_values:
+  kubeProxyReplacement: true
+  k8sServiceHost: api-int.${openshift:baseDomain}
+  k8sServicePort: "6443"
+  k8sClientRateLimit:
+    qps: 35 <1>
+    burst: 45 <2>
+----
+<1> Setting the base QPS rate.
+<2> The burst QPS should be slightly higher.
+
+=== `l2_announcements.policies`
+
+[horizontal]
+type:: object
+default:: `{}`
+
+This parameter allows users to deploy `CiliumL2AnnouncementPolicy` resources.
+
+Each key-value pair in the parameter is converted to a `CiliumL2AnnouncementPolicy` resource.
+Entries can be removed by setting the value to `null`.
+
+See https://docs.cilium.io/en/latest/network/l2-announcements/#policies[the upstream documentation] for further explanation.
+
+==== Example
+
+[source,yaml]
+----
+l2_announcements:
+  policies:
+    color_blue:
+      spec:
+        serviceSelector:
+          matchLabels:
+            color: blue
+        nodeSelector:
+          matchExpressions:
+            - key: node-role.kubernetes.io/control-plane
+              operator: DoesNotExist
+        interfaces:
+        - ^eth[0-9]+
+        externalIPs: true
+        loadBalancerIPs: true
+----
+
 == `bgp`
 
 This section allows users to configure the https://docs.cilium.io/en/stable/network/bgp-control-plane/[Cilium BGP control plane].

tests/golden/bgp-control-plane/cilium/cilium/01_cilium_helmchart/cilium/templates/cilium-configmap.yaml

@@ -71,8 +71,8 @@ data:
   install-no-conntrack-iptables-rules: 'false'
   ipam: cluster-pool
   ipam-cilium-node-update-rate: 15s
-  k8s-client-burst: '20'
-  k8s-client-qps: '10'
+  k8s-client-burst: '30'
+  k8s-client-qps: '15'
   kube-proxy-replacement: 'true'
   kube-proxy-replacement-healthz-bind-address: ''
   max-connected-clusters: '255'

tests/golden/defaults/cilium/cilium/01_cilium_helmchart/cilium/templates/cilium-configmap.yaml

@@ -70,8 +70,8 @@ data:
   install-no-conntrack-iptables-rules: 'false'
   ipam: cluster-pool
   ipam-cilium-node-update-rate: 15s
-  k8s-client-burst: '20'
-  k8s-client-qps: '10'
+  k8s-client-burst: '30'
+  k8s-client-qps: '15'
   kube-proxy-replacement: 'true'
   kube-proxy-replacement-healthz-bind-address: ''
   max-connected-clusters: '255'

tests/golden/egress-gateway/cilium/cilium/01_cilium_helmchart/cilium/templates/cilium-configmap.yaml

@@ -71,8 +71,8 @@ data:
   install-no-conntrack-iptables-rules: 'false'
   ipam: cluster-pool
   ipam-cilium-node-update-rate: 15s
-  k8s-client-burst: '20'
-  k8s-client-qps: '10'
+  k8s-client-burst: '30'
+  k8s-client-qps: '15'
   kube-proxy-replacement: 'true'
   kube-proxy-replacement-healthz-bind-address: ''
   max-connected-clusters: '255'

tests/golden/helm-opensource/cilium/cilium/01_cilium_helmchart/cilium/templates/cilium-configmap.yaml

@@ -70,8 +70,8 @@ data:
   install-no-conntrack-iptables-rules: 'false'
   ipam: cluster-pool
   ipam-cilium-node-update-rate: 15s
-  k8s-client-burst: '20'
-  k8s-client-qps: '10'
+  k8s-client-burst: '30'
+  k8s-client-qps: '15'
   kube-proxy-replacement: 'true'
   kube-proxy-replacement-healthz-bind-address: ''
   max-connected-clusters: '255'

tests/golden/kubeproxyreplacement-strict/cilium/cilium/01_cilium_helmchart/cilium/templates/cilium-configmap.yaml

@@ -70,8 +70,8 @@ data:
   install-no-conntrack-iptables-rules: 'false'
   ipam: cluster-pool
   ipam-cilium-node-update-rate: 15s
-  k8s-client-burst: '20'
-  k8s-client-qps: '10'
+  k8s-client-burst: '30'
+  k8s-client-qps: '15'
   kube-proxy-replacement: 'true'
   kube-proxy-replacement-healthz-bind-address: ''
   max-connected-clusters: '255'

tests/golden/l2-announcement/cilium/cilium/01_cilium_helmchart/cilium/templates/cilium-agent/clusterrole.yaml

@@ -33,6 +33,16 @@ rules:
       - get
       - list
       - watch
+  - apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+    verbs:
+      - create
+      - get
+      - update
+      - list
+      - delete
   - apiGroups:
       - apiextensions.k8s.io
     resources:

tests/golden/l2-announcement/cilium/cilium/01_cilium_helmchart/cilium/templates/cilium-configmap.yaml

@@ -43,6 +43,7 @@ data:
   enable-ipv6-masquerade: 'true'
   enable-k8s-networkpolicy: 'true'
   enable-k8s-terminating-endpoint: 'true'
+  enable-l2-announcements: 'true'
   enable-l2-neigh-discovery: 'true'
   enable-l7-proxy: 'true'
   enable-local-redirect-policy: 'false'
@@ -70,8 +71,8 @@ data:
   install-no-conntrack-iptables-rules: 'false'
   ipam: cluster-pool
   ipam-cilium-node-update-rate: 15s
-  k8s-client-burst: '20'
-  k8s-client-qps: '10'
+  k8s-client-burst: '45'
+  k8s-client-qps: '35'
   kube-proxy-replacement: 'true'
   kube-proxy-replacement-healthz-bind-address: ''
   max-connected-clusters: '255'

tests/golden/l2-announcement/cilium/cilium/40_l2_announcement_policies.yaml

@@ -0,0 +1,20 @@
+apiVersion: cilium.io/v2alpha1
+kind: CiliumL2AnnouncementPolicy
+metadata:
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true,Prune=false
+  labels:
+    name: color-blue
+  name: color-blue
+spec:
+  externalIPs: true
+  interfaces:
+    - ^eth[0-9]+
+  loadBalancerIPs: true
+  nodeSelector:
+    matchExpressions:
+      - key: node-role.kubernetes.io/control-plane
+        operator: DoesNotExist
+  serviceSelector:
+    matchLabels:
+      color: blue

tests/golden/olm-opensource/cilium/cilium/olm/cluster-network-07-cilium-ciliumconfig.yaml

@@ -35,9 +35,14 @@ spec:
       clusterPoolIPv4MaskSize: '23'
       clusterPoolIPv4PodCIDRList:
         - 10.128.0.0/14
+  k8sClientRateLimit:
+    burst: 30
+    qps: 15
   k8sServiceHost: 172.30.0.1
   k8sServicePort: 443
   kubeProxyReplacement: 'true'
+  l2announcements:
+    enabled: false
   l7Proxy: true
   operator:
     prometheus:

tests/l2-announcement.yml

@@ -1,3 +1,23 @@
-# Overwrite parameters here
+parameters:
+  cilium:
+    cilium_helm_values:
+      k8sClientRateLimit:
+        qps: 35
+        burst: 45
 
-# parameters: {...}
+    l2_announcements:
+      enabled: true
+      policies:
+        color_blue:
+          spec:
+            serviceSelector:
+              matchLabels:
+                color: blue
+            nodeSelector:
+              matchExpressions:
+                - key: node-role.kubernetes.io/control-plane
+                  operator: DoesNotExist
+            interfaces:
+              - ^eth[0-9]+
+            externalIPs: true
+            loadBalancerIPs: true