Merge pull request #308 from luxas/improve_cni

Improve the CNI implementation, and documentation

README.md

@@ -156,6 +156,7 @@ Please refer to the following documents:
 - [Scope and Dependencies](https://ignite.readthedocs.io/en/latest/dependencies.html)
 - [Getting Started Walkthrough](https://ignite.readthedocs.io/en/latest/usage.html)
 - [Declaratively Controlling Ignite](https://ignite.readthedocs.io/en/latest/declarative-config.html)
+- [Networking and Multi-Node](https://ignite.readthedocs.io/en/latest/networking.html)
 - [CLI Reference](https://ignite.readthedocs.io/en/latest/cli/ignite/)
 - [API Reference](https://ignite.readthedocs.io/en/latest/api/)
 

docs/networking.md

@@ -0,0 +1,68 @@
+# Networking
+
+The default networking mode is `docker-bridge`, which means that the default docker bridge will be used for the networking setup.
+The default docker bridge is a local `docker0` interface, giving out local addresses to containers in the `172.17.0.0/16` range.
+
+Ignite also supports integration with [CNI](https://github.com/containernetworking/cni), the standard networking interface
+for Kubernetes and many other cloud-native projects and container runtimes. Note that CNI itself is only an interface, not
+an implementation, so if you use this mode you need to install an implementation of this interface. Any implementation that works
+with Kubernetes should technically work with Ignite.
+
+## Comparison
+
+### docker-bridge
+
+**Pros:**
+
+- **Quick start**: If you're running docker, you can get up and running without installing extra software
+- **Port mapping support**: This mode supports port mappings from the VM to the host
+
+**Cons:**
+
+- **docker-dependent**: By design, this mode is can only be used with docker, and is hence not portable across container runtimes.
+- **No multi-node support**: The IP is local (in the `172.17.0.0/16` range), and hence other computers can't connect to your VM's IP address.
+
+### CNI
+
+**Pros:**
+
+- **Multi-node support**: CNI implementations can often route packets between multiple physical hosts. External computers can access the VM's IP.
+- **Kubernetes-compatible**: You can use the same overlay networks as you use with Kubernetes, and hence get your VMs on the same network as your containers.
+
+**Cons:**
+
+- **More software needed**: There's now one extra piece of software to install and manage.
+- **No port-mapping support** (yet): For the moment, we haven't implemented port mapping support for this mode.
+
+## Multi-node networking with Weave Net
+
+To use e.g. Ignite together with [Weave Net](https://github.com/weaveworks/weave), run this on all physical machines that
+need to connect to the overlay network:
+
+```shell
+# This tries to autodetect the primary IP address of this machine
+# Ref: https://stackoverflow.com/questions/13322485/how-to-get-the-primary-ip-address-of-the-local-machine-on-linux-and-macos
+export PRIMARY_IP_ADDRESS=$(ip -o route get 1.1.1.1 | cut -d' ' -f7)
+# A space-separated list of all the peers in the overlay network
+export KUBE_PEERS="${PRIMARY_IP_ADDRESS}"
+# Start Weave Net in a container
+docker run -d \
+  --privileged \
+  --net host \
+  --pid host \
+  --restart always \
+  -e HOSTNAME="$(hostname)" \
+  -e KUBE_PEERS="${KUBE_PEERS}" \
+  -v /var/lib/weave:/weavedb \
+  -v /opt:/host/opt \
+  -v /home:/host/home \
+  -v /etc:/host/etc \
+  -v /var/lib/dbus:/host/var/lib/dbus \
+  -v /lib/modules:/lib/modules \
+  -v /run/xtables.lock:/run/xtables.lock \
+  --entrypoint /home/weave/launch.sh \
+  weaveworks/weave-kube:2.5.2
+```
+
+If you're running Kubernetes on the physical machine you want to use for Ignite VMs, it should work out of the box, as
+the CNI implementation is most probably already running in a `DaemonSet` on that machine.

pkg/network/cni/cni.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/containernetworking/cni/libcni"
 	cnitypes "github.com/containernetworking/cni/pkg/types"
+	cnicurrentapi "github.com/containernetworking/cni/pkg/types/current"
 	log "github.com/sirupsen/logrus"
 	"github.com/weaveworks/ignite/pkg/network"
 	"github.com/weaveworks/ignite/pkg/runtime"
@@ -162,22 +163,46 @@ func (plugin *cniNetworkPlugin) Status() error {
 	return plugin.checkInitialized()
 }
 
-func (plugin *cniNetworkPlugin) SetupContainerNetwork(containerid string) error {
+func (plugin *cniNetworkPlugin) PrepareContainerSpec(container *runtime.ContainerConfig) error {
+	// No need for the container runtime to set up networking, as this plugin will do it
+	container.NetworkMode = "none"
+	return nil
+}
+
+func (plugin *cniNetworkPlugin) SetupContainerNetwork(containerid string) (*network.Result, error) {
 	if err := plugin.checkInitialized(); err != nil {
-		return err
+		return nil, err
 	}
 
 	netnsPath, err := plugin.runtime.ContainerNetNS(containerid)
 	if err != nil {
-		return fmt.Errorf("CNI failed to retrieve network namespace path: %v", err)
+		return nil, fmt.Errorf("CNI failed to retrieve network namespace path: %v", err)
 	}
 
 	if _, err = plugin.addToNetwork(plugin.loNetwork, containerid, netnsPath); err != nil {
-		return err
+		return nil, err
+	}
+
+	genericResult, err := plugin.addToNetwork(plugin.getDefaultNetwork(), containerid, netnsPath)
+	if err != nil {
+		return nil, err
+	}
+	result, err := cnicurrentapi.NewResultFromResult(genericResult)
+	if err != nil {
+		return nil, err
 	}
+	return cniToIgniteResult(result), nil
+}
 
-	_, err = plugin.addToNetwork(plugin.getDefaultNetwork(), containerid, netnsPath)
-	return err
+func cniToIgniteResult(r *cnicurrentapi.Result) *network.Result {
+	result := &network.Result{}
+	for _, ip := range r.IPs {
+		result.Addresses = append(result.Addresses, network.Address{
+			IPNet:   ip.Address,
+			Gateway: ip.Gateway,
+		})
+	}
+	return result
 }
 
 func (plugin *cniNetworkPlugin) RemoveContainerNetwork(containerid string) error {

pkg/network/docker/docker.go

@@ -0,0 +1,59 @@
+package docker
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/weaveworks/ignite/pkg/network"
+	"github.com/weaveworks/ignite/pkg/runtime"
+)
+
+const pluginName = "docker-bridge"
+
+type dockerNetworkPlugin struct {
+	runtime runtime.Interface
+}
+
+func GetDockerNetworkPlugin(r runtime.Interface) network.Plugin {
+	return &dockerNetworkPlugin{r}
+}
+
+func (*dockerNetworkPlugin) Name() string {
+	return pluginName
+}
+
+func (*dockerNetworkPlugin) PrepareContainerSpec(_ *runtime.ContainerConfig) error {
+	// no-op, we don't need to set any special parameters on the container
+	return nil
+}
+
+func (plugin *dockerNetworkPlugin) SetupContainerNetwork(containerID string) (*network.Result, error) {
+	// This is used to fetch the IP address the runtime gives to the VM container
+	result, err := plugin.runtime.InspectContainer(containerID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to inspect container %s: %v", containerID, err)
+	}
+
+	return &network.Result{
+		Addresses: []network.Address{
+			{
+				IPNet: net.IPNet{
+					IP:   result.IPAddress,
+					Mask: net.IPv4Mask(255, 255, 0, 0),
+				},
+				// TODO: Make this auto-detect if the gateway is not using the standard setup
+				Gateway: net.IPv4(result.IPAddress[0], result.IPAddress[1], result.IPAddress[2], 1),
+			},
+		},
+	}, nil
+}
+
+func (*dockerNetworkPlugin) RemoveContainerNetwork(_ string) error {
+	// no-op for docker, this is handled automatically
+	return nil
+}
+
+func (*dockerNetworkPlugin) Status() error {
+	// no-op, we assume the bridge to be working :)
+	return nil
+}

pkg/network/types.go

@@ -1,16 +1,35 @@
 package network
 
+import (
+	"net"
+
+	"github.com/weaveworks/ignite/pkg/runtime"
+)
+
 // Plugin describes a generic network plugin
 type Plugin interface {
 	// Name returns the network plugin's name.
 	Name() string
 
+	// PrepareContainerSpec sets any needed options on the container spec before starting the container
+	PrepareContainerSpec(container *runtime.ContainerConfig) error
+
 	// SetupContainerNetwork sets up the networking for a container
-	SetupContainerNetwork(containerID string) error
+	// This is ran _after_ the container has been started
+	SetupContainerNetwork(containerID string) (*Result, error)
 
 	// RemoveContainerNetwork is the method called before a container using the network plugin can be deleted
 	RemoveContainerNetwork(containerID string) error
 
 	// Status returns error if the network plugin is in error state
 	Status() error
 }
+
+type Result struct {
+	Addresses []Address
+}
+
+type Address struct {
+	net.IPNet
+	Gateway net.IP
+}

pkg/operations/remove.go

@@ -60,8 +60,8 @@ func RemoveVMContainer(vm meta.Object) error {
 		return fmt.Errorf("failed to remove container for VM %q: %v", vm.GetUID(), err)
 	}
 
-	// Remove the CNI networking of the VM
-	return removeCNINetworking(vm.(*api.VM), result.ID)
+	// Tear down the networking of the VM
+	return removeNetworking(vm.(*api.VM), result.ID)
 }
 
 // StopVM stops or kills a VM
@@ -95,13 +95,9 @@ func StopVM(vm *api.VM, kill, silent bool) error {
 	return nil
 }
 
-func removeCNINetworking(vm *api.VM, containerID string) error {
-	// Skip all other network modes
-	if vm.Spec.Network.Mode != api.NetworkModeCNI {
-		return nil
-	}
-
+func removeNetworking(vm *api.VM, containerID string) error {
 	// Perform the removal
-	log.Infof("Trying to remove the container with ID %q from the CNI network", containerID)
-	return providers.NetworkPlugin.RemoveContainerNetwork(containerID)
+	networkPlugin := providers.NetworkPlugins[vm.Spec.Network.Mode.String()]
+	log.Debugf("Removing the container with ID %q from the %s network", networkPlugin.Name(), containerID)
+	return networkPlugin.RemoveContainerNetwork(containerID)
 }

pkg/operations/start.go

@@ -92,9 +92,11 @@ func StartVM(vm *api.VM, debug bool) error {
 		})
 	}
 
-	// If the VM is using CNI networking, disable Docker's own implementation
-	if vm.Spec.Network.Mode == api.NetworkModeCNI {
-		config.NetworkMode = "none"
+	networkPlugin := providers.NetworkPlugins[vm.Spec.Network.Mode.String()]
+
+	// Prepare the networking for the container, for the given network plugin
+	if err := networkPlugin.PrepareContainerSpec(config); err != nil {
+		return err
 	}
 
 	// If we're not debugging, remove the container post-run
@@ -108,21 +110,13 @@ func StartVM(vm *api.VM, debug bool) error {
 		return fmt.Errorf("failed to start container for VM %q: %v", vm.GetUID(), err)
 	}
 
-	// Set the container ID for the VM
-	vm.Status.Runtime = &api.Runtime{ID: containerID}
-
-	// Set the start time for the VM
-	startTime := meta.Timestamp()
-	vm.Status.StartTime = &startTime
-
-	if vm.Spec.Network.Mode == api.NetworkModeCNI {
-		if err := providers.NetworkPlugin.SetupContainerNetwork(containerID); err != nil {
-			return err
-		}
-
-		log.Infof("Networking is now handled by CNI")
+	// Set up the networking
+	result, err := networkPlugin.SetupContainerNetwork(containerID)
+	if err != nil {
+		return err
 	}
 
+	log.Infof("Networking is handled by %q", networkPlugin.Name())
 	log.Infof("Started Firecracker VM %q in a container with ID %q", vm.GetUID(), containerID)
 
 	// TODO: Follow-up the container here with a defer, or dedicated goroutine. We should output
@@ -132,15 +126,17 @@ func StartVM(vm *api.VM, debug bool) error {
 		return err
 	}
 
-	// This is used to fetch the IP address the runtime gives to the VM container
-	// TODO: This needs to be handled differently for CNI, the IP address will be blank
-	result, err := providers.Runtime.InspectContainer(containerID)
-	if err != nil {
-		return fmt.Errorf("failed to inspect container for VM %q: %v", vm.GetUID(), err)
-	}
+	// Set the container ID for the VM
+	vm.Status.Runtime = &api.Runtime{ID: containerID}
+
+	// Set the start time for the VM
+	startTime := meta.Timestamp()
+	vm.Status.StartTime = &startTime
 
 	// Append the runtime IP address of the VM to its state
-	vm.Status.IPAddresses = append(vm.Status.IPAddresses, result.IPAddress)
+	for _, addr := range result.Addresses {
+		vm.Status.IPAddresses = append(vm.Status.IPAddresses, addr.IP)
+	}
 
 	// Set the VM's status to running
 	vm.Status.Running = true

pkg/providers/cni/cni.go

@@ -8,6 +8,10 @@ import (
 
 func SetCNINetworkPlugin() (err error) {
 	log.Trace("Initializing the CNI provider...")
-	providers.NetworkPlugin, err = cni.GetCNINetworkPlugin(providers.Runtime)
+	plugin, err := cni.GetCNINetworkPlugin(providers.Runtime)
+	if err != nil {
+		return err
+	}
+	providers.NetworkPlugins[plugin.Name()] = plugin
 	return
 }

pkg/providers/docker/runtime.go

@@ -2,12 +2,20 @@ package docker
 
 import (
 	log "github.com/sirupsen/logrus"
+	network "github.com/weaveworks/ignite/pkg/network/docker"
 	"github.com/weaveworks/ignite/pkg/providers"
-	"github.com/weaveworks/ignite/pkg/runtime/docker"
+	runtime "github.com/weaveworks/ignite/pkg/runtime/docker"
 )
 
 func SetDockerRuntime() (err error) {
-	log.Trace("Initializing the Docker provider...")
-	providers.Runtime, err = docker.GetDockerClient()
+	log.Trace("Initializing the Docker runtime provider...")
+	providers.Runtime, err = runtime.GetDockerClient()
 	return
 }
+
+func SetDockerNetwork() error {
+	log.Trace("Initializing the Docker network provider...")
+	plugin := network.GetDockerNetworkPlugin(providers.Runtime)
+	providers.NetworkPlugins[plugin.Name()] = plugin
+	return nil
+}

pkg/providers/ignite/providers.go

@@ -12,6 +12,7 @@ import (
 // E.g. the network plugin depends on the runtime.
 var Providers = []providers.ProviderInitFunc{
 	dockerprovider.SetDockerRuntime,   // Use the Docker runtime
+	dockerprovider.SetDockerNetwork,   // Use the Docker bridge network
 	cniprovider.SetCNINetworkPlugin,   // Use the CNI Network plugin
 	storageprovider.SetGenericStorage, // Use a generic storage implementation backed by a cache
 	clientprovider.SetClient,          // Set the globally available client

pkg/providers/ignited/providers.go

@@ -12,6 +12,7 @@ import (
 // E.g. the network plugin depends on the runtime.
 var Providers = []providers.ProviderInitFunc{
 	dockerprovider.SetDockerRuntime,            // Use the Docker runtime
+	dockerprovider.SetDockerNetwork,            // Use the Docker bridge network
 	cniprovider.SetCNINetworkPlugin,            // Use the CNI Network plugin
 	manifeststorageprovider.SetManifestStorage, // Use the ManifestStorage implementation, backed by a cache
 	clientprovider.SetClient,                   // Set the globally available client

pkg/providers/providers.go

@@ -8,8 +8,8 @@ import (
 	"github.com/weaveworks/ignite/pkg/storage"
 )
 
-// NetworkPlugin provides the default network plugin implementation
-var NetworkPlugin network.Plugin
+// NetworkPlugins provides the initialized network plugins indexed by their name
+var NetworkPlugins = make(map[string]network.Plugin)
 
 // Runtime provides the default container runtime
 var Runtime runtime.Interface