pkg/: FEATURE: support allowed IPs outside a cluster

cmd/kgctl/graph.go

@@ -60,7 +60,7 @@ func runGraph(_ *cobra.Command, _ []string) error {
 			peers[p.Name] = p
 		}
 	}
-	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, 0, []byte{}, subnet, nodes[hostname].PersistentKeepalive)
+	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, 0, []byte{}, subnet, nodes[hostname].PersistentKeepalive, nil)
 	if err != nil {
 		return fmt.Errorf("failed to create topology: %v", err)
 	}

cmd/kgctl/showconf.go

@@ -147,7 +147,7 @@ func runShowConfNode(_ *cobra.Command, args []string) error {
 		}
 	}
 
-	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, opts.port, []byte{}, subnet, nodes[hostname].PersistentKeepalive)
+	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, opts.port, []byte{}, subnet, nodes[hostname].PersistentKeepalive, nil)
 	if err != nil {
 		return fmt.Errorf("failed to create topology: %v", err)
 	}
@@ -236,7 +236,7 @@ func runShowConfPeer(_ *cobra.Command, args []string) error {
 		return fmt.Errorf("did not find any peer named %q in the cluster", peer)
 	}
 
-	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, mesh.DefaultKiloPort, []byte{}, subnet, peers[peer].PersistentKeepalive)
+	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, mesh.DefaultKiloPort, []byte{}, subnet, peers[peer].PersistentKeepalive, nil)
 	if err != nil {
 		return fmt.Errorf("failed to create topology: %v", err)
 	}

docs/annotations.md

@@ -9,6 +9,7 @@ The following annotations can be added to any Kubernetes Node object to configur
 |[kilo.squat.ai/leader](#leader)|string|`""`, `true`|
 |[kilo.squat.ai/location](#location)|string|`gcp-east`, `lab`|
 |[kilo.squat.ai/persistent-keepalive](#persistent-keepalive)|uint|`10`|
+|[kilo.squat.ai/allowed-location-ips](#allowed-location-ips)|CIDR|`66.66.66.66/32`|
 
 ### force-endpoint
 In order to create links between locations, Kilo requires at least one node in each location to have an endpoint, ie a `host:port` combination, that is routable from the other locations.
@@ -52,3 +53,10 @@ In order for a node behind NAT to receive packets from nodes outside of the NATe
 The frequency of emission of these keepalive packets can be controlled by setting the persistent-keepalive annotation on the node behind NAT.
 The annotated node will use the specified value will as the persistent-keepalive interval for all of its peers.
 For more background, [see the WireGuard documentation on NAT and firewall traversal](https://www.wireguard.com/quickstart/#nat-and-firewall-traversal-persistence).
+
+### allowed-location-ips
+It is possible to add allowed-location-ips to a location by annotating any node within that location.
+Adding allowed-location-ips to a location makes these IPs routable from other locations as well.
+
+In an example deployment of Kilo with two locations A and B, a printer in location A can be accessible from nodes and pods in location B.
+Additionally, Kilo Peers can use the printer in location A.

pkg/k8s/backend.go

@@ -59,6 +59,7 @@ const (
 	persistentKeepaliveKey       = "kilo.squat.ai/persistent-keepalive"
 	wireGuardIPAnnotationKey     = "kilo.squat.ai/wireguard-ip"
 	discoveredEndpointsKey       = "kilo.squat.ai/discovered-endpoints"
+	allowedLocationIPsKey        = "kilo.squat.ai/allowed-location-ips"
 	// RegionLabelKey is the key for the well-known Kubernetes topology region label.
 	RegionLabelKey  = "topology.kubernetes.io/region"
 	jsonPatchSlash  = "~1"
@@ -311,6 +312,15 @@ func translateNode(node *v1.Node, topologyLabel string) *mesh.Node {
 			discoveredEndpoints = nil
 		}
 	}
+	// Set allowed IPs for a location.
+	var allowedLocationIPs []*net.IPNet
+	if str, ok := node.ObjectMeta.Annotations[allowedLocationIPsKey]; ok {
+		for _, ip := range strings.Split(str, ",") {
+			if ipnet := normalizeIP(ip); ipnet != nil {
+				allowedLocationIPs = append(allowedLocationIPs, ipnet)
+			}
+		}
+	}
 
 	return &mesh.Node{
 		// Endpoint and InternalIP should only ever fail to parse if the
@@ -334,6 +344,7 @@ func translateNode(node *v1.Node, topologyLabel string) *mesh.Node {
 		// will parse as nil.
 		WireGuardIP:         normalizeIP(node.ObjectMeta.Annotations[wireGuardIPAnnotationKey]),
 		DiscoveredEndpoints: discoveredEndpoints,
+		AllowedLocationIPs:  allowedLocationIPs,
 	}
 }
 

pkg/mesh/backend.go

@@ -67,6 +67,7 @@ type Node struct {
 	Subnet              *net.IPNet
 	WireGuardIP         *net.IPNet
 	DiscoveredEndpoints map[string]*wireguard.Endpoint
+	AllowedLocationIPs  []*net.IPNet
 }
 
 // Ready indicates whether or not the node is ready.

pkg/mesh/mesh.go

@@ -380,6 +380,7 @@ func (m *Mesh) handleLocal(n *Node) {
 		Subnet:              n.Subnet,
 		WireGuardIP:         m.wireGuardIP,
 		DiscoveredEndpoints: n.DiscoveredEndpoints,
+		AllowedLocationIPs:  n.AllowedLocationIPs,
 	}
 	if !nodesAreEqual(n, local) {
 		level.Debug(m.logger).Log("msg", "local node differs from backend")
@@ -460,7 +461,7 @@ func (m *Mesh) applyTopology() {
 	oldConf := wireguard.Parse(oldConfRaw)
 	natEndpoints := discoverNATEndpoints(nodes, peers, oldConf, m.logger)
 	nodes[m.hostname].DiscoveredEndpoints = natEndpoints
-	t, err := NewTopology(nodes, peers, m.granularity, m.hostname, nodes[m.hostname].Endpoint.Port, m.priv, m.subnet, nodes[m.hostname].PersistentKeepalive)
+	t, err := NewTopology(nodes, peers, m.granularity, m.hostname, nodes[m.hostname].Endpoint.Port, m.priv, m.subnet, nodes[m.hostname].PersistentKeepalive, m.logger)
 	if err != nil {
 		level.Error(m.logger).Log("error", err)
 		m.errorCounter.WithLabelValues("apply").Inc()
@@ -674,7 +675,7 @@ func nodesAreEqual(a, b *Node) bool {
 	// Ignore LastSeen when comparing equality we want to check if the nodes are
 	// equivalent. However, we do want to check if LastSeen has transitioned
 	// between valid and invalid.
-	return string(a.Key) == string(b.Key) && ipNetsEqual(a.WireGuardIP, b.WireGuardIP) && ipNetsEqual(a.InternalIP, b.InternalIP) && a.Leader == b.Leader && a.Location == b.Location && a.Name == b.Name && subnetsEqual(a.Subnet, b.Subnet) && a.Ready() == b.Ready() && a.PersistentKeepalive == b.PersistentKeepalive && discoveredEndpointsAreEqual(a.DiscoveredEndpoints, b.DiscoveredEndpoints)
+	return string(a.Key) == string(b.Key) && ipNetsEqual(a.WireGuardIP, b.WireGuardIP) && ipNetsEqual(a.InternalIP, b.InternalIP) && a.Leader == b.Leader && a.Location == b.Location && a.Name == b.Name && subnetsEqual(a.Subnet, b.Subnet) && a.Ready() == b.Ready() && a.PersistentKeepalive == b.PersistentKeepalive && discoveredEndpointsAreEqual(a.DiscoveredEndpoints, b.DiscoveredEndpoints) && ipNetSlicesEqual(a.AllowedLocationIPs, b.AllowedLocationIPs)
 }
 
 func peersAreEqual(a, b *Peer) bool {
@@ -713,6 +714,18 @@ func ipNetsEqual(a, b *net.IPNet) bool {
 	return a.IP.Equal(b.IP)
 }
 
+func ipNetSlicesEqual(a, b []*net.IPNet) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if !ipNetsEqual(a[i], b[i]) {
+			return false
+		}
+	}
+	return true
+}
+
 func subnetsEqual(a, b *net.IPNet) bool {
 	if a == nil && b == nil {
 		return true

pkg/mesh/routes.go

@@ -108,6 +108,17 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
 					Protocol:  unix.RTPROT_STATIC,
 				}, enc.Strategy(), t.privateIP, tunlIface))
 			}
+			// For segments / locations other than the location of this instance of kg,
+			// we need to set routes for allowed location IPs over the leader in the current location.
+			for i := range segment.allowedLocationIPs {
+				routes = append(routes, encapsulateRoute(&netlink.Route{
+					Dst:       segment.allowedLocationIPs[i],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        gw,
+					LinkIndex: privIface,
+					Protocol:  unix.RTPROT_STATIC,
+				}, enc.Strategy(), t.privateIP, tunlIface))
+			}
 		}
 		// Add routes for the allowed IPs of peers.
 		for _, peer := range t.peers {
@@ -198,6 +209,17 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface
 				Protocol:  unix.RTPROT_STATIC,
 			})
 		}
+		// For segments / locations other than the location of this instance of kg,
+		// we need to set routes for allowed location IPs over the wg interface.
+		for i := range segment.allowedLocationIPs {
+			routes = append(routes, &netlink.Route{
+				Dst:       segment.allowedLocationIPs[i],
+				Flags:     int(netlink.FLAG_ONLINK),
+				Gw:        segment.wireGuardIP,
+				LinkIndex: kiloIface,
+				Protocol:  unix.RTPROT_STATIC,
+			})
+		}
 	}
 	// Add routes for the allowed IPs of peers.
 	for _, peer := range t.peers {
@@ -232,6 +254,16 @@ func (t *Topology) Rules(cni bool) []iptables.Rule {
 		for _, aip := range s.allowedIPs {
 			rules = append(rules, iptables.NewRule(iptables.GetProtocol(len(aip.IP)), "nat", "KILO-NAT", "-d", aip.String(), "-m", "comment", "--comment", "Kilo: do not NAT packets destined for known IPs", "-j", "RETURN"))
 		}
+		// Make sure packets to allowed location IPs go through the KILO-NAT chain, so they can be MASQUERADEd,
+		// Otherwise packets to these destinations will reach the destination, but never find their way back.
+		// We only want to NAT in locations of the corresponding allowed location IPs.
+		if t.location == s.location {
+			for _, alip := range s.allowedLocationIPs {
+				rules = append(rules,
+					iptables.NewRule(iptables.GetProtocol(len(alip.IP)), "nat", "POSTROUTING", "-d", alip.String(), "-m", "comment", "--comment", "Kilo: jump to NAT chain", "-j", "KILO-NAT"),
+				)
+			}
+		}
 	}
 	for _, p := range t.peers {
 		for _, aip := range p.AllowedIPs {

pkg/mesh/routes_test.go

@@ -74,6 +74,13 @@ func TestRoutes(t *testing.T) {
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
+				{
+					Dst:       nodes["b"].AllowedLocationIPs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
 				{
 					Dst:       mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[2].cidrs[0],
 					Flags:     int(netlink.FLAG_ONLINK),
@@ -258,6 +265,13 @@ func TestRoutes(t *testing.T) {
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
+				{
+					Dst:       nodes["b"].AllowedLocationIPs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["d"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
 				{
 					Dst:       peers["a"].AllowedIPs[0],
 					LinkIndex: kiloIface,
@@ -294,6 +308,13 @@ func TestRoutes(t *testing.T) {
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
+				{
+					Dst:       nodes["b"].AllowedLocationIPs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
 				{
 					Dst:       mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[2].cidrs[0],
 					Flags:     int(netlink.FLAG_ONLINK),
@@ -422,6 +443,13 @@ func TestRoutes(t *testing.T) {
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
+				{
+					Dst:       nodes["b"].AllowedLocationIPs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
 				{
 					Dst:       mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[3].cidrs[0],
 					Flags:     int(netlink.FLAG_ONLINK),
@@ -480,6 +508,13 @@ func TestRoutes(t *testing.T) {
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
+				{
+					Dst:       nodes["b"].AllowedLocationIPs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
 				{
 					Dst:       nodes["d"].Subnet,
 					Flags:     int(netlink.FLAG_ONLINK),
@@ -538,6 +573,13 @@ func TestRoutes(t *testing.T) {
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
+				{
+					Dst:       nodes["b"].AllowedLocationIPs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(LogicalGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
 				{
 					Dst:       nodes["d"].Subnet,
 					Flags:     int(netlink.FLAG_ONLINK),
@@ -875,6 +917,13 @@ func TestRoutes(t *testing.T) {
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
+				{
+					Dst:       nodes["b"].AllowedLocationIPs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["a"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
 				{
 					Dst:       nodes["c"].Subnet,
 					Flags:     int(netlink.FLAG_ONLINK),
@@ -1005,6 +1054,13 @@ func TestRoutes(t *testing.T) {
 					LinkIndex: kiloIface,
 					Protocol:  unix.RTPROT_STATIC,
 				},
+				{
+					Dst:       nodes["b"].AllowedLocationIPs[0],
+					Flags:     int(netlink.FLAG_ONLINK),
+					Gw:        mustTopoForGranularityAndHost(FullGranularity, nodes["c"].Name).segments[1].wireGuardIP,
+					LinkIndex: kiloIface,
+					Protocol:  unix.RTPROT_STATIC,
+				},
 				{
 					Dst:       nodes["d"].Subnet,
 					Flags:     int(netlink.FLAG_ONLINK),