Skip to content

Commit

Permalink
coordinator: add a from policy route for pod's eth0
Browse files Browse the repository at this point in the history 
Add a from policy route for pod's eth0, which make sure that packets received from eth0 are forwarded out of eth0. fix to the problem of inconsistent routes.

Signed-off-by: cyclinder <qifeng.guo@daocloud.io>
  • Loading branch information
@cyclinder
cyclinder committed Jul 2, 2024
1 parent 5566f50 commit 95ee34d
Show file tree
Hide file tree
Showing 3 changed files with 311 additions and 72 deletions.
24 changes: 24 additions & 0 deletions cmd/coordinator/cmd/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -486,6 +486,30 @@ func (c *coordinator) tunePodRoutes(logger *zap.Logger, configDefaultRouteNIC st
return err
}
}

if c.tuneMode == ModeOverlay && c.firstInvoke {
// mv calico or cilium default route to table 500 to fix to the problem of
// inconsistent routes, the pod forwards the response packet from net1 (macvlan)
// when it sends the response packet. but the request packet comes in eth0(calico).
// see https://github.com/spidernet-io/spiderpool/issues/3683

// copy to table 500,
podOverlayDefaultRouteRuleTable := c.hostRuleTable
for idx := range defaultInterfaceAddress {
ipNet := networking.ConvertMaxMaskIPNet(defaultInterfaceAddress[idx].IP)
err = networking.AddFromRuleTable(ipNet, podOverlayDefaultRouteRuleTable)
if err != nil {
logger.Error("failed to AddFromRuleTable", zap.Error(err))
return err
}
}

// move all routes of the specified interface to a new route table
if err = networking.CopyDefaultRoute(logger, defaultOverlayVethName, unix.RT_TABLE_MAIN, podOverlayDefaultRouteRuleTable, c.ipFamily); err != nil {
return err
}
}

}
// move all routes of the specified interface to a new route table
if err = networking.MoveRouteTable(logger, moveRouteInterface, unix.RT_TABLE_MAIN, c.currentRuleTable, c.ipFamily); err != nil {
Expand Down
187 changes: 128 additions & 59 deletions pkg/networking/networking/route.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,18 +149,57 @@ func AddRoute(logger *zap.Logger, ruleTable, ipFamily int, scope netlink.Scope,
return nil
}

// MoveRouteTable move all routes of the specified interface to a new route table
// Equivalent: `ip route del <route>` and `ip r route add <route> <table>`
func MoveRouteTable(logger *zap.Logger, iface string, srcRuleTable, dstRuleTable, ipfamily int) error {
logger.Debug("Debug MoveRouteTable", zap.String("interface", iface),
zap.Int("srcRuleTable", srcRuleTable), zap.Int("dstRuleTable", dstRuleTable))
func GetLinkIndexAndRoutes(iface string, ipfamily int) (int, []netlink.Route, error) {
link, err := netlink.LinkByName(iface)
if err != nil {
return -1, nil, err
}

routes, err := netlink.RouteList(nil, ipfamily)
if err != nil {
return -1, nil, err
}

return link.Attrs().Index, routes, nil
}

// CopyDefaultRoute found the default route of pod's eth0 nic, and copy this
// to dstRuleTable.
func CopyDefaultRoute(logger *zap.Logger, iface string, srcRuleTable, podOverlayDefaultRouteRuleTable, ipfamily int) error {
logger.Debug("Debug MoveRouteTable", zap.String("interface", iface),
zap.Int("srcRuleTable", srcRuleTable), zap.Int("dstRuleTable", podOverlayDefaultRouteRuleTable))

linkIndex, routes, err := GetLinkIndexAndRoutes(iface, ipfamily)
if err != nil {
logger.Error(err.Error())
return err
}

routes, err := netlink.RouteList(nil, ipfamily)
for _, route := range routes {
// only handle route tables from table main
if route.Table != srcRuleTable {
continue
}

// ignore local link route
if route.Dst.String() == "fe80::/64" {
continue
}

if err = moveRouteTable(linkIndex, srcRuleTable, podOverlayDefaultRouteRuleTable, true, route, logger); err != nil {
return err
}

}
return nil
}

// MoveRouteTable move all routes of the specified interface to a new route table
// Equivalent: `ip route del <route>` and `ip r route add <route> <table>`
func MoveRouteTable(logger *zap.Logger, iface string, srcRuleTable, dstRuleTable, ipfamily int) error {
logger.Debug("Debug MoveRouteTable", zap.String("interface", iface),
zap.Int("srcRuleTable", srcRuleTable), zap.Int("dstRuleTable", dstRuleTable))
linkIndex, routes, err := GetLinkIndexAndRoutes(iface, ipfamily)
if err != nil {
logger.Error(err.Error())
return err
Expand All @@ -177,72 +216,102 @@ func MoveRouteTable(logger *zap.Logger, iface string, srcRuleTable, dstRuleTable
continue
}

if route.LinkIndex == link.Attrs().Index {
// only delete default route
if route.Dst == nil || route.Dst.IP.Equal(net.IPv4zero) || route.Dst.IP.Equal(net.IPv6zero) {
if err = netlink.RouteDel(&route); err != nil {
logger.Error("failed to RouteDel in main", zap.String("route", route.String()), zap.Error(err))
return fmt.Errorf("failed to RouteDel %s in main table: %+v", route.String(), err)
}
logger.Debug("Del the default route from main successfully", zap.String("Route", route.String()))
}
if err = moveRouteTable(linkIndex, srcRuleTable, dstRuleTable, false, route, logger); err != nil {
return err
}

}
return nil
}

// we need copy the all routes in main table of the podDefaultRouteNic to dstRuleTable.
// Otherwise, the reply packet don't know
// moveRouteTable move route table from srcRuleTable to dstRuleTable. NOTE: if copyOverlayDefaultRoute is true,
// only add the default route to host rule table and exit in advance.
func moveRouteTable(linkIndex, srcRuleTable, dstRuleTable int, onlyCopyOverlayDefaultRoute bool, route netlink.Route, logger *zap.Logger) error {
var err error
if route.LinkIndex == linkIndex {
if route.Dst == nil || route.Dst.IP.Equal(net.IPv4zero) || route.Dst.IP.Equal(net.IPv6zero) {
route.Table = dstRuleTable
if err = netlink.RouteAdd(&route); err != nil && !os.IsExist(err) {
logger.Error("failed to RouteAdd in new table ", zap.String("route", route.String()), zap.Error(err))
logger.Error("failed to copy overlay default route to hostRuleTable", zap.String("route", route.String()), zap.Error(err))
return fmt.Errorf("failed to RouteAdd (%+v) to new table: %+v", route, err)
}
logger.Debug("MoveRoute to new table successfully", zap.String("Route", route.String()))
} else {
// in high kernel, if pod has multi ipv6 default routes, all default routes
// will be put in MultiPath
/*
{
Gw: [{Ifindex: 3 Weight: 1 Gw: fd00:10:7::103 Flags: []} {Ifindex: 5 Weight: 1 Gw: fd00:10:6::100 Flags: []}]}"
}
*/
if len(route.MultiPath) == 0 {
continue
}
logger.Debug("Copy the overlay default route to hostRuleTable successfully", zap.String("Route", route.String()))

var generatedRoute, deletedRoute *netlink.Route
// get generated default Route for new table
for _, v := range route.MultiPath {
logger.Debug("Found IPv6 Default Route", zap.String("Route", route.String()),
zap.Int("v.LinkIndex", v.LinkIndex), zap.Int("link.Attrs().Index", link.Attrs().Index))
if v.LinkIndex == link.Attrs().Index {
generatedRoute = &netlink.Route{
LinkIndex: v.LinkIndex,
Gw: v.Gw,
Table: dstRuleTable,
MTU: route.MTU,
}
deletedRoute = &netlink.Route{
LinkIndex: v.LinkIndex,
Gw: v.Gw,
Table: srcRuleTable,
}
break
}
}
if generatedRoute == nil {
continue
if onlyCopyOverlayDefaultRoute {
// only copy overlay default route, don't neet delete the default route
return nil
}

logger.Debug("Deleting IPv6 DefaultRoute", zap.String("deletedRoute", deletedRoute.String()))
if err := netlink.RouteDel(deletedRoute); err != nil {
logger.Error("failed to RouteDel for IPv6", zap.String("Route", route.String()), zap.Error(err))
return fmt.Errorf("failed to RouteDel %v for IPv6: %+v", route.String(), err)
// Del the default route from main
route.Table = srcRuleTable
if err = netlink.RouteDel(&route); err != nil {
logger.Error("failed to RouteDel in main", zap.String("route", route.String()), zap.Error(err))
return fmt.Errorf("failed to RouteDel %s in main table: %+v", route.String(), err)
}
logger.Debug("Del the default route from main successfully", zap.String("Route", route.String()))
}

if onlyCopyOverlayDefaultRoute {
// only copy overlay default route, don't neet add non-default routes
return nil
}

// we need copy the all routes in main table of the podDefaultRouteNic to dstRuleTable.
// Otherwise, the reply packet don't know
if err = netlink.RouteAdd(&route); err != nil && !os.IsExist(err) {
logger.Error("failed to RouteAdd in new table ", zap.String("route", route.String()), zap.Error(err))
return fmt.Errorf("failed to RouteAdd (%+v) to new table: %+v", route, err)
}
logger.Debug("MoveRoute to new table successfully", zap.String("Route", route.String()))
return nil
}

if err = netlink.RouteAdd(generatedRoute); err != nil && !os.IsExist(err) {
logger.Error("failed to RouteAdd for IPv6 to new table", zap.String("route", route.String()), zap.Error(err))
return fmt.Errorf("failed to RouteAdd for IPv6 (%+v) to new table: %+v", route.String(), err)
// in high kernel, if pod has multi ipv6 default routes, all default routes
// will be put in MultiPath
/*
{
Gw: [{Ifindex: 3 Weight: 1 Gw: fd00:10:7::103 Flags: []} {Ifindex: 5 Weight: 1 Gw: fd00:10:6::100 Flags: []}]}"
}
*/
if len(route.MultiPath) == 0 {
return nil
}

var generatedRoute, deletedRoute *netlink.Route
// get generated default Route for new table
for _, v := range route.MultiPath {
logger.Debug("Found IPv6 Default Route", zap.String("Route", route.String()),
zap.Int("v.LinkIndex", linkIndex), zap.Int("link.Attrs().Index", linkIndex))
if v.LinkIndex == linkIndex {
generatedRoute = &netlink.Route{
LinkIndex: v.LinkIndex,
Gw: v.Gw,
Table: dstRuleTable,
MTU: route.MTU,
}
deletedRoute = &netlink.Route{
LinkIndex: v.LinkIndex,
Gw: v.Gw,
Table: srcRuleTable,
}
break
}
}

if generatedRoute == nil || onlyCopyOverlayDefaultRoute {
return nil
}

logger.Debug("Deleting IPv6 DefaultRoute", zap.String("deletedRoute", deletedRoute.String()))
if err := netlink.RouteDel(deletedRoute); err != nil {
logger.Error("failed to RouteDel for IPv6", zap.String("Route", route.String()), zap.Error(err))
return fmt.Errorf("failed to RouteDel %v for IPv6: %+v", route.String(), err)
}

if err = netlink.RouteAdd(generatedRoute); err != nil && !os.IsExist(err) {
logger.Error("failed to RouteAdd for IPv6 to new table", zap.String("route", route.String()), zap.Error(err))
return fmt.Errorf("failed to RouteAdd for IPv6 (%+v) to new table: %+v", route.String(), err)
}
return nil
}

Expand Down
Loading

0 comments on commit 95ee34d

Please sign in to comment.