Modify networking to enable integration with K3s

pkg/pillar/base/kubevirt.go

@@ -0,0 +1,27 @@
+// Copyright (c) 2023 Zededa, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package base
+
+import (
+	"os"
+	"strings"
+)
+
+const (
+	// EveVirtTypeFile contains the virtualization type, ie kvm, xen or kubevirt
+	EveVirtTypeFile = "/run/eve-hv-type"
+)
+
+// IsHVTypeKube - return true if the EVE image is kube cluster type.
+func IsHVTypeKube() bool {
+	retbytes, err := os.ReadFile(EveVirtTypeFile)
+	if err != nil {
+		return false
+	}
+
+	if strings.Contains(string(retbytes), "kubevirt") {
+		return true
+	}
+	return false
+}

pkg/pillar/cmd/nim/nim.go

@@ -181,6 +181,7 @@ func (n *nim) init() (err error) {
 		SubEdgeNodeCert:      n.subEdgeNodeCert,
 		PubCipherBlockStatus: n.pubCipherBlockStatus,
 		CipherMetrics:        n.cipherMetrics,
+		HVTypeKube:           base.IsHVTypeKube(),
 	}
 	n.dpcManager = &dpcmanager.DpcManager{
 		Log:                      n.Log,

pkg/pillar/cmd/nodeagent/nodeagent.go

@@ -115,6 +115,7 @@ type nodeagentContext struct {
 	maintModeReason             types.MaintenanceModeReason //reason for entering Maintenance mode
 	configGetSuccess            bool                        // got config from controller success
 	vaultmgrReported            bool                        // got reports from vaultmgr
+	hvTypeKube                  bool                        // image is kubernetes cluster type
 
 	// Some constants.. Declared here as variables to enable unit tests
 	minRebootDelay          uint32
@@ -142,6 +143,7 @@ func newNodeagentContext(_ *pubsub.PubSub, _ *logrus.Logger, _ *base.LogObject)
 	curpart := agentlog.EveCurrentPartition()
 	nodeagentCtx.curPart = strings.TrimSpace(curpart)
 	nodeagentCtx.vaultOperational = types.TS_NONE
+	nodeagentCtx.hvTypeKube = base.IsHVTypeKube()
 	return &nodeagentCtx
 }
 
@@ -697,6 +699,7 @@ func publishNodeAgentStatus(ctxPtr *nodeagentContext) {
 		RestartCounter:             ctxPtr.restartCounter,
 		LocalMaintenanceMode:       ctxPtr.maintMode,
 		LocalMaintenanceModeReason: ctxPtr.maintModeReason,
+		HVTypeKube:                 ctxPtr.hvTypeKube,
 	}
 	ctxPtr.lastLock.Unlock()
 	pub.Publish(agentName, status)

pkg/pillar/dpcreconciler/linux.go

@@ -141,6 +141,12 @@ const (
 	intendedStateFile = "/run/nim-intended-state.dot"
 )
 
+const (
+	// Network bridge used by Kubernetes CNI.
+	// Currently, this is hardcoded for the Flannel CNI plugin.
+	kubeCNIBridge = "cni0"
+)
+
 // LinuxDpcReconciler is a DPC-reconciler for Linux network stack,
 // i.e. it configures and uses Linux networking to provide device connectivity.
 type LinuxDpcReconciler struct {
@@ -180,6 +186,8 @@ type LinuxDpcReconciler struct {
 	prevArgs     Args
 	prevStatus   ReconcileStatus
 	radioSilence types.RadioSilence
+
+	HVTypeKube bool
 }
 
 type pendingReconcile struct {
@@ -721,6 +729,17 @@ func (r *LinuxDpcReconciler) updateCurrentAdapterAddrs(
 func (r *LinuxDpcReconciler) updateCurrentRoutes(dpc types.DevicePortConfig) (changed bool) {
 	sgPath := dg.NewSubGraphPath(L3SG, RoutesSG)
 	currentRoutes := dg.New(dg.InitArgs{Name: RoutesSG})
+	cniIfIndex := -1
+	if r.HVTypeKube {
+		ifIndex, found, err := r.NetworkMonitor.GetInterfaceIndex(kubeCNIBridge)
+		if err != nil {
+			r.Log.Errorf("getIntendedRoutes: failed to get ifIndex for %s: %v",
+				kubeCNIBridge, err)
+		}
+		if err == nil && found {
+			cniIfIndex = ifIndex
+		}
+	}
 	for _, port := range dpc.Ports {
 		ifIndex, found, err := r.NetworkMonitor.GetInterfaceIndex(port.IfName)
 		if err != nil {
@@ -741,7 +760,6 @@ func (r *LinuxDpcReconciler) updateCurrentRoutes(dpc types.DevicePortConfig) (ch
 		if err != nil {
 			r.Log.Errorf("updateCurrentRoutes: ListRoutes failed for ifIndex %d: %v",
 				ifIndex, err)
-			continue
 		}
 		for _, rt := range routes {
 			currentRoutes.PutItem(linux.Route{
@@ -753,6 +771,29 @@ func (r *LinuxDpcReconciler) updateCurrentRoutes(dpc types.DevicePortConfig) (ch
 				LastOperation: reconciler.OperationCreate,
 			})
 		}
+
+		if cniIfIndex != -1 {
+			cniRoutes, err := r.NetworkMonitor.ListRoutes(netmonitor.RouteFilters{
+				FilterByTable: true,
+				Table:         table,
+				FilterByIf:    true,
+				IfIndex:       cniIfIndex,
+			})
+			if err != nil {
+				r.Log.Errorf("updateCurrentRoutes: ListRoutes failed for ifIndex %d: %v",
+					cniIfIndex, err)
+			}
+			for _, rt := range cniRoutes {
+				currentRoutes.PutItem(linux.Route{
+					Route:         rt.Data.(netlink.Route),
+					UnmanagedLink: true,
+				}, &reconciler.ItemStateData{
+					State:         reconciler.ItemStateCreated,
+					LastOperation: reconciler.OperationCreate,
+				})
+			}
+
+		}
 	}
 	prevSG := dg.GetSubGraph(r.currentState, sgPath)
 	if len(prevSG.DiffItems(currentRoutes)) > 0 {
@@ -1003,6 +1044,28 @@ func (r *LinuxDpcReconciler) getIntendedRoutes(dpc types.DevicePortConfig) dg.Gr
 		Description: "IP routes",
 	}
 	intendedRoutes := dg.New(graphArgs)
+	// Routes are copied from the main table.
+	srcTable := syscall.RT_TABLE_MAIN
+	var cniRoutes []netmonitor.Route
+	if r.HVTypeKube {
+		ifIndex, found, err := r.NetworkMonitor.GetInterfaceIndex(kubeCNIBridge)
+		if err != nil {
+			r.Log.Errorf("getIntendedRoutes: failed to get ifIndex for %s: %v",
+				kubeCNIBridge, err)
+		}
+		if err == nil && found {
+			cniRoutes, err = r.NetworkMonitor.ListRoutes(netmonitor.RouteFilters{
+				FilterByTable: true,
+				Table:         srcTable,
+				FilterByIf:    true,
+				IfIndex:       ifIndex,
+			})
+			if err != nil {
+				r.Log.Errorf("getIntendedRoutes: ListRoutes failed for ifIndex %d: %v",
+					ifIndex, err)
+			}
+		}
+	}
 	for _, port := range dpc.Ports {
 		ifIndex, found, err := r.NetworkMonitor.GetInterfaceIndex(port.IfName)
 		if err != nil {
@@ -1013,8 +1076,6 @@ func (r *LinuxDpcReconciler) getIntendedRoutes(dpc types.DevicePortConfig) dg.Gr
 		if !found {
 			continue
 		}
-		// Routes copied from the main table.
-		srcTable := syscall.RT_TABLE_MAIN
 		dstTable := devicenetwork.DPCBaseRTIndex + ifIndex
 		routes, err := r.NetworkMonitor.ListRoutes(netmonitor.RouteFilters{
 			FilterByTable: true,
@@ -1025,31 +1086,43 @@ func (r *LinuxDpcReconciler) getIntendedRoutes(dpc types.DevicePortConfig) dg.Gr
 		if err != nil {
 			r.Log.Errorf("getIntendedRoutes: ListRoutes failed for ifIndex %d: %v",
 				ifIndex, err)
-			continue
 		}
 		for _, rt := range routes {
 			rtCopy := rt.Data.(netlink.Route)
 			rtCopy.Table = dstTable
-			// Multiple IPv6 link-locals can't be added to the same
-			// table unless the Priority differs.
-			// Different LinkIndex, Src, Scope doesn't matter.
-			if rt.Dst != nil && rt.Dst.IP.IsLinkLocalUnicast() {
-				if r.Log != nil {
-					r.Log.Tracef("Forcing IPv6 priority to %v", rtCopy.LinkIndex)
-				}
-				// Hack to make the kernel routes not appear identical.
-				rtCopy.Priority = rtCopy.LinkIndex
-			}
+			r.prepareRouteForCopy(&rtCopy)
 			intendedRoutes.PutItem(linux.Route{
 				Route:         rtCopy,
 				AdapterIfName: port.IfName,
 				AdapterLL:     port.Logicallabel,
 			}, nil)
 		}
+		for _, rt := range cniRoutes {
+			rtCopy := rt.Data.(netlink.Route)
+			rtCopy.Table = dstTable
+			r.prepareRouteForCopy(&rtCopy)
+			intendedRoutes.PutItem(linux.Route{
+				Route:         rtCopy,
+				UnmanagedLink: true,
+			}, nil)
+		}
 	}
 	return intendedRoutes
 }
 
+func (r *LinuxDpcReconciler) prepareRouteForCopy(route *netlink.Route) {
+	// Multiple IPv6 link-locals can't be added to the same
+	// table unless the Priority differs.
+	// Different LinkIndex, Src, Scope doesn't matter.
+	if route.Dst != nil && route.Dst.IP.IsLinkLocalUnicast() {
+		if r.Log != nil {
+			r.Log.Tracef("Forcing IPv6 priority to %v", route.LinkIndex)
+		}
+		// Hack to make the kernel routes not appear identical.
+		route.Priority = route.LinkIndex
+	}
+}
+
 type portAddr struct {
 	logicalLabel string
 	ifName       string
@@ -1505,9 +1578,23 @@ func (r *LinuxDpcReconciler) getIntendedACLs(
 		TargetOpts:  []string{"--set-mark", iptables.ControlProtocolMarkingIDMap["in_dhcp"]},
 		Description: "Mark ingress DHCP traffic",
 	}
+	// Allow kubernetes DNS replies from an external server.
+	// XXX Maybe there is a better way to setup this, like using set-mark for outbound
+	// kubernetes DNS queries.
+	markDNS := iptables.Rule{
+		RuleLabel:   "Incoming DNS replies",
+		MatchOpts:   []string{"-p", "udp", "--sport", "domain"},
+		Target:      "CONNMARK",
+		TargetOpts:  []string{"--set-mark", iptables.ControlProtocolMarkingIDMap["in_dns"]},
+		Description: "Incoming DNS replies (used to allow kubernetes DNS replies from external server)",
+	}
+
 	protoMarkV4Rules := []iptables.Rule{
 		markSSHAndGuacamole, markVnc, markIcmpV4, markDhcp,
 	}
+	if r.HVTypeKube {
+		protoMarkV4Rules = append(protoMarkV4Rules, markDNS)
+	}
 	protoMarkV6Rules := []iptables.Rule{
 		markSSHAndGuacamole, markVnc, markIcmpV6,
 	}

pkg/pillar/dpcreconciler/linuxitems/route.go

@@ -26,9 +26,17 @@ type Route struct {
 	netlink.Route
 	// AdapterIfName : name of the interface associated with the route.
 	// Should match with Route.LinkIndex.
+	// Empty if this route is referencing unmanaged link.
 	AdapterIfName string
 	// AdapterLL : logical label of the associated interface.
+	// Empty if this route is referencing unmanaged link.
 	AdapterLL string
+	// True if this route is referencing link not managed by EVE.
+	// It is used by DPC Reconciler to install routes for the bridge cni0 created
+	// and used by Flannel CNI.
+	// Later this will likely go away as we find cleaner solution for integration
+	// between EVE and (possibly any) CNI plugin.
+	UnmanagedLink bool
 }
 
 // Name combines the interface name, route table ID and the destination
@@ -40,8 +48,12 @@ func (r Route) Name() string {
 	} else {
 		dst = r.Route.Dst.String()
 	}
+	ifName := r.AdapterIfName
+	if r.UnmanagedLink {
+		ifName = fmt.Sprintf("unmanaged-%d", r.LinkIndex)
+	}
 	return fmt.Sprintf("%d/%s/%s",
-		r.Table, r.AdapterIfName, dst)
+		r.Table, ifName, dst)
 }
 
 // Label is more human-readable than name.
@@ -52,8 +64,14 @@ func (r Route) Label() string {
 	} else {
 		dst = r.Route.Dst.String()
 	}
+	var dev string
+	if r.UnmanagedLink {
+		dev = fmt.Sprintf("<unmanaged-%d>", r.LinkIndex)
+	} else {
+		dev = r.AdapterLL
+	}
 	return fmt.Sprintf("%s route table %d dst %s dev %v via %v",
-		r.ipVersionStr(), r.Table, dst, r.AdapterLL, r.Gw)
+		r.ipVersionStr(), r.Table, dst, dev, r.Gw)
 }
 
 func (r Route) ipVersionStr() string {
@@ -120,6 +138,11 @@ func (r Route) External() bool {
 
 // String describes the network route.
 func (r Route) String() string {
+	if r.UnmanagedLink {
+		return fmt.Sprintf(
+			"Network route for unmanaged link (index=%d) with priority %d: %s",
+			r.LinkIndex, r.Route.Priority, r.Route.String())
+	}
 	return fmt.Sprintf("Network route for adapter '%s' with priority %d: %s",
 		r.AdapterLL, r.Route.Priority, r.Route.String())
 }
@@ -129,6 +152,11 @@ func (r Route) String() string {
 //   - the "via" adapter must have an IP address assigned from the subnet
 //     of the route gateway.
 func (r Route) Dependencies() (deps []depgraph.Dependency) {
+	if r.UnmanagedLink {
+		// Reconciler does not check dependencies for route associated
+		// with link not managed by EVE.
+		return nil
+	}
 	return []depgraph.Dependency{
 		{
 			RequiredItem: depgraph.ItemRef{

pkg/pillar/iptables/connmark.go

@@ -54,6 +54,8 @@ var ControlProtocolMarkingIDMap = map[string]string{
 	"app_http": "11",
 	// ICMPv6 traffic to and from an application
 	"app_icmpv6": "12",
+	// for Kubernetes DNS, allowing coreDNS to talk to external DNS servers
+	"in_dns": "13",
 }
 
 // GetConnmark : create connection mark corresponding to the given attributes.

pkg/pillar/types/zedagenttypes.go

@@ -428,6 +428,7 @@ type NodeAgentStatus struct {
 	RebootImage                string
 	LocalMaintenanceMode       bool                  //enter Maintenance Mode
 	LocalMaintenanceModeReason MaintenanceModeReason //reason for Maintenance Mode
+	HVTypeKube                 bool
 }
 
 // Key :