Skip to content
This repository has been archived by the owner on Dec 7, 2023. It is now read-only.

Enable multiple non-IP interface to be connected via tc redirect #836

Merged
merged 28 commits into from
Jun 25, 2021
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
0d7046e
First crack at multinet implementation
networkop May 10, 2021
0b3dfdb
fix the dhcp server bug
networkop May 10, 2021
9acef4f
Merge branch 'weaveworks:main' into multinet
networkop May 10, 2021
56ca546
Added wait for X number of interfaces to be connected
networkop May 10, 2021
52776a6
Merge branch 'weaveworks:main' into multinet
networkop May 10, 2021
e3baa91
Merge branch 'multinet' of github.com:networkop/ignite into multinet
networkop May 10, 2021
17765be
Fixed multiple bugs
networkop May 11, 2021
6e19436
Ensure maxIntfs is at least 1
networkop May 11, 2021
39d395e
Fixed formatting
networkop May 11, 2021
9a31fb9
Forgot to assign envVars
networkop May 11, 2021
20b4aa8
Updated docs
networkop May 11, 2021
35dc891
More updated docs
networkop May 11, 2021
e8bd47c
More docs updates
networkop May 11, 2021
b8ef5d3
Not using env vars anymore
networkop May 22, 2021
47c9140
Re-introducing env var support
networkop May 23, 2021
6f1f14a
Reverting the order to error checks
networkop May 24, 2021
90eaa58
Added multinet tests
networkop Jun 9, 2021
1020f90
Fixing dox
networkop Jun 9, 2021
053def5
Refactored StartVM with async spawn channel
networkop Jun 10, 2021
eb8997f
Refactored ignite-spawn's network setup
networkop Jun 10, 2021
051ad2b
Fixed the CNI plugin bug
networkop Jun 11, 2021
1c70e78
Tested with containerlab
networkop Jun 12, 2021
9957b70
Merging suggested changes
networkop Jun 21, 2021
3161ef4
Alternative e2e test
networkop Jun 22, 2021
f8d361d
Removed --wait flag and fixed up multinet tests
networkop Jun 22, 2021
668b5fe
Removing waitForSSH from exec
networkop Jun 24, 2021
71c801c
Removing leftover comment
networkop Jun 24, 2021
abddda2
Adding waitforSSH back
networkop Jun 24, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fixed the CNI plugin bug
  • Loading branch information
networkop committed Jun 11, 2021
commit 051ad2bf5b0c259a30653ee4566439c500892db3
6 changes: 3 additions & 3 deletions pkg/constants/vm.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,9 @@ const (
// image to be used.
DEFAULT_SANDBOX_IMAGE_TAG = "dev"

// IGNITE_INTERFACE_ANNOTATION is the annotation key to store a list of extra interfaces
// IGNITE_INTERFACE_ANNOTATION is the annotation prefix to store a list of extra interfaces
IGNITE_INTERFACE_ANNOTATION = "ignite.weave.works/interface/"

// IGNITE_SANDBOX_ENV_VAR is the annotation key to store a list of env variables
IGNITE_SANDBOX_ENV_VAR = "ignite.weave.works/sanbox-env"
// IGNITE_SANDBOX_ENV_VAR is the annotation prefix to store a list of env variables
IGNITE_SANDBOX_ENV_VAR = "ignite.weave.works/sanbox-env/"
)
77 changes: 50 additions & 27 deletions pkg/container/network.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,12 +100,12 @@ func collectInterfaces(vmIntfs map[string]string) (bool, error) {
}

// create a map of candidate interfaces
foundIntfs := make(map[string]struct{})
foundIntfs := make(map[string]*net.Interface)
for _, intf := range allIntfs {
if _, ok := ignoreInterfaces[intf.Name]; ok {
continue
}
foundIntfs[intf.Name] = struct{}{}
foundIntfs[intf.Name] = &intf

// default fallback behaviour to always consider intfs with an address
addrs, _ := intf.Addrs()
Expand All @@ -119,6 +119,18 @@ func collectInterfaces(vmIntfs map[string]string) (bool, error) {
if _, ok := foundIntfs[intfName]; !ok {
return true, fmt.Errorf("interface %q (mode %q) is still not found", intfName, mode)
}

// for DHCP interface, we need to make sure IP and route exist
if mode == supportedModes.dhcp {
_, _, _, noIPs, err := getAddress(foundIntfs[intfName])
if err != nil {
return true, err
}

if noIPs {
return true, fmt.Errorf("IP is still not found on %q", intfName)
}
}
}
return false, nil
}
Expand All @@ -145,13 +157,10 @@ func networkSetup(fcIntfs *firecracker.NetworkInterfaces, dhcpIntfs *[]DHCPInter

switch vmIntfs[intfName] {
case supportedModes.dhcp:
ipNet, gw, noIPs, err := takeAddress(intf)
ipNet, gw, err := takeAddress(intf)
if err != nil {
return fmt.Errorf("error parsing interface %q: %s", intfName, err)
}
if noIPs {
return fmt.Errorf("interface %q expected to have an IP but none found", intfName)
}

dhcpIface, err := bridge(intf)
if err != nil {
Expand Down Expand Up @@ -305,12 +314,13 @@ func bridge(iface *net.Interface) (*DHCPInterface, error) {
}, nil
}

// takeAddress removes the first address of an interface and returns it and the appropriate gateway
func takeAddress(iface *net.Interface) (*net.IPNet, *net.IP, bool, error) {
// getAddress collects the first IP and gateway information from an interface
// in case of multiple routes over an interface, only the first one is considered
func getAddress(iface *net.Interface) (*net.IPNet, *net.IP, netlink.Link, bool, error) {
addrs, err := iface.Addrs()
if err != nil || addrs == nil || len(addrs) == 0 {
// set the bool to true so the caller knows to retry
return nil, nil, true, fmt.Errorf("interface %q has no address", iface.Name)
return nil, nil, nil, true, fmt.Errorf("interface %q has no address", iface.Name)
}

for _, addr := range addrs {
Expand All @@ -337,42 +347,55 @@ func takeAddress(iface *net.Interface) (*net.IPNet, *net.IP, bool, error) {

link, err := netlink.LinkByName(iface.Name)
if err != nil {
return nil, nil, false, fmt.Errorf("failed to get interface %q by name: %v", iface.Name, err)
return nil, nil, nil, false, fmt.Errorf("failed to get interface %q by name: %v", iface.Name, err)
}

var gw *net.IP
gwString := "<nil>"
routes, err := netlink.RouteList(link, netlink.FAMILY_ALL)
if err != nil {
return nil, nil, false, fmt.Errorf("failed to get default gateway for interface %q: %v", iface.Name, err)
return nil, nil, nil, false, fmt.Errorf("failed to get default gateway for interface %q: %v", iface.Name, err)
}
for _, rt := range routes {
if rt.Gw != nil {
gw = &rt.Gw
gwString = gw.String()
break
}
}

delAddr := &netlink.Addr{
IPNet: &net.IPNet{
IP: ip,
Mask: mask,
},
}
if err = netlink.AddrDel(link, delAddr); err != nil {
return nil, nil, false, fmt.Errorf("failed to remove address %q from interface %q: %v", delAddr, iface.Name, err)
}

log.Infof("Moving IP address %s (%s) with gateway %s from container to VM", ip.String(), maskString(mask), gwString)

return &net.IPNet{
IP: ip,
Mask: mask,
}, gw, false, nil
}, gw, link, false, nil
}

return nil, nil, true, fmt.Errorf("interface %s has no valid addresses", iface.Name)
return nil, nil, nil, true, fmt.Errorf("interface %q has no valid addresses", iface.Name)
}

// takeAddress removes the first address of an interface and returns it and the appropriate gateway
func takeAddress(iface *net.Interface) (*net.IPNet, *net.IP, error) {

ip, gw, link, noIPs, err := getAddress(iface)
if err != nil {
return nil, nil, err
}
if noIPs {
return nil, nil, fmt.Errorf("interface %q expected to have an IP but none found", iface.Name)
}

delAddr := &netlink.Addr{
IPNet: &net.IPNet{
IP: ip.IP,
Mask: ip.Mask,
},
}

if err = netlink.AddrDel(link, delAddr); err != nil {
return nil, nil, fmt.Errorf("failed to remove address %q from interface %q: %v", delAddr, iface.Name, err)
}

log.Infof("Moving IP address %s (%s) with gateway %s from container to VM", ip.String(), maskString(ip.Mask), gw.String())

return ip, gw, nil
}

// createTAPAdapter creates a new TAP device with the given name
Expand Down
10 changes: 4 additions & 6 deletions pkg/operations/start.go
Original file line number Diff line number Diff line change
Expand Up @@ -105,14 +105,12 @@ func StartVMNonBlocking(vm *api.VM, debug bool) (*VMChannels, error) {
PortBindings: vm.Spec.Network.Ports, // Add the port mappings to Docker
}

// Only set envVars if at least one was configured
var envVars []string
parts := strings.Split(vm.GetAnnotation((constants.IGNITE_SANDBOX_ENV_VAR)), ",")
for _, part := range parts {
if part == "" {
continue
for k, v := range vm.GetObjectMeta().Annotations {
if strings.HasPrefix(k, constants.IGNITE_SANDBOX_ENV_VAR) {
k := strings.TrimPrefix(k, constants.IGNITE_SANDBOX_ENV_VAR)
envVars = append(envVars, fmt.Sprintf("%s=%s", k, v))
}
envVars = append(envVars, part)
}
config.EnvVars = envVars

Expand Down