Merge pull request #836 from networkop/multinet

cmd/ignite-spawn/ignite-spawn.go

@@ -39,8 +39,9 @@ func decodeVM(vmID string) (*api.VM, error) {
 }
 
 func StartVM(vm *api.VM) (err error) {
+
 	// Setup networking inside of the container, return the available interfaces
-	dhcpIfaces, err := container.SetupContainerNetworking()
+	fcIfaces, dhcpIfaces, err := container.SetupContainerNetworking(vm)
 	if err != nil {
 		return fmt.Errorf("network setup failed: %v", err)
 	}
@@ -66,7 +67,7 @@ func StartVM(vm *api.VM) (err error) {
 	defer util.DeferErr(&err, func() error { return os.Remove(metricsSocket) })
 
 	// Execute Firecracker
-	if err = container.ExecuteFirecracker(vm, dhcpIfaces); err != nil {
+	if err = container.ExecuteFirecracker(vm, fcIfaces); err != nil {
 		return fmt.Errorf("runtime error for VM %q: %v", vm.GetUID(), err)
 	}
 

cmd/ignite/run/exec.go

@@ -2,6 +2,7 @@ package run
 
 import (
 	api "github.com/weaveworks/ignite/pkg/apis/ignite"
+	"github.com/weaveworks/ignite/pkg/constants"
 )
 
 // ExecFlags contains the flags supported by the exec command.
@@ -30,5 +31,8 @@ func (ef *ExecFlags) NewExecOptions(vmMatch string, command ...string) (eo *Exec
 
 // Exec executes command in a VM based on the provided ExecOptions.
 func Exec(eo *ExecOptions) error {
+	if err := waitForSSH(eo.vm, constants.SSH_DEFAULT_TIMEOUT_SECONDS, int(eo.Timeout)); err != nil {
+		return err
+	}
 	return runSSH(eo.vm, eo.IdentityFile, eo.command, eo.Tty, eo.Timeout)
 }

e2e/multinet_test.go

@@ -0,0 +1,268 @@
+package e2e
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/weaveworks/ignite/e2e/util"
+	api "github.com/weaveworks/ignite/pkg/apis/ignite"
+	meta "github.com/weaveworks/ignite/pkg/apis/meta/v1alpha1"
+	igniteConstants "github.com/weaveworks/ignite/pkg/constants"
+	"github.com/weaveworks/ignite/pkg/dmlegacy"
+	"github.com/weaveworks/ignite/pkg/metadata"
+	"github.com/weaveworks/ignite/pkg/network"
+	"github.com/weaveworks/ignite/pkg/operations"
+	"github.com/weaveworks/ignite/pkg/providers"
+	igniteDocker "github.com/weaveworks/ignite/pkg/providers/docker"
+	"github.com/weaveworks/ignite/pkg/providers/ignite"
+	"github.com/weaveworks/ignite/pkg/runtime"
+	igniteUtil "github.com/weaveworks/ignite/pkg/util"
+	"gotest.tools/assert"
+)
+
+var (
+	multinetVM  = "e2e-test-vm-multinet"
+	sanboxImage = "weaveworks/ignite:dev"
+	kernelImage = "weaveworks/ignite-kernel:5.4.108"
+	vmImage     = "weaveworks/ignite-ubuntu"
+)
+
+func startAsyncVM(t *testing.T, intfs []string) (*operations.VMChannels, string) {
+
+	assert.Assert(t, e2eHome != "", "IGNITE_E2E_HOME should be set")
+	igniteUtil.GenericCheckErr(providers.Populate(ignite.Preload))
+
+	_ = igniteDocker.SetDockerRuntime()
+	_ = igniteDocker.SetDockerNetwork()
+
+	providers.RuntimeName = runtime.RuntimeDocker
+	providers.NetworkPluginName = network.PluginDockerBridge
+	_ = providers.Populate(ignite.Providers)
+
+	vm := providers.Client.VMs().New()
+	vm.Status.Runtime.Name = runtime.RuntimeDocker
+	vm.Status.Network.Plugin = network.PluginDockerBridge
+
+	ociRef, err := meta.NewOCIImageRef(sanboxImage)
+	if err != nil {
+		t.Fatalf("Failed to parse OCI image ref %s: %s", sanboxImage, err)
+	}
+	vm.Spec.Sandbox.OCI = ociRef
+
+	ociRef, err = meta.NewOCIImageRef(kernelImage)
+	if err != nil {
+		t.Fatalf("Failed to parse OCI image ref %s: %s", kernelImage, err)
+	}
+	vm.Spec.Kernel.OCI = ociRef
+	k, _ := operations.FindOrImportKernel(providers.Client, ociRef)
+	vm.SetKernel(k)
+
+	ociRef, err = meta.NewOCIImageRef(vmImage)
+	if err != nil {
+		t.Fatalf("Failed to parse OCI image ref %s: %s", vmImage, err)
+	}
+	img, err := operations.FindOrImportImage(providers.Client, ociRef)
+	if err != nil {
+		t.Fatalf("Failed to find OCI image ref %s: %s", ociRef, err)
+	}
+	vm.SetImage(img)
+
+	vm.Name = multinetVM
+	vm.Spec.SSH = &api.SSH{Generate: true}
+
+	_ = metadata.SetNameAndUID(vm, providers.Client)
+
+	for _, intf := range intfs {
+		vm.SetAnnotation(igniteConstants.IGNITE_INTERFACE_ANNOTATION+intf, "tc-redirect")
+	}
+
+	_ = providers.Client.VMs().Set(vm)
+
+	err = dmlegacy.AllocateAndPopulateOverlay(vm)
+	if err != nil {
+		t.Fatalf("Error AllocateAndPopulateOverlay: %s", err)
+	}
+
+	vmChans, err := operations.StartVMNonBlocking(vm, false)
+	if err != nil {
+		t.Fatalf("failed to start a VM: \n%q\n", err)
+	}
+
+	return vmChans, vm.GetUID().String()
+}
+
+// TestMultipleInterface tests that a VM's can be configured with more than 1 interface
+func TestOneExtraInterface(t *testing.T) {
+	assert.Assert(t, e2eHome != "", "IGNITE_E2E_HOME should be set")
+
+	igniteCmd := util.NewCommand(t, igniteBin)
+	dockerCmd := util.NewCommand(t, runtime.RuntimeDocker.String())
+
+	vmChans, vmID := startAsyncVM(t, []string{"foo"})
+
+	// Clean-up the following VM.
+	defer igniteCmd.New().
+		With("rm", "-f", multinetVM).
+		Run()
+
+	fooAddr := "aa:ca:e9:12:34:56"
+	dockerCmd.New().
+		With("exec", fmt.Sprintf("ignite-%s", vmID)).
+		With("ip", "link", "add", "foo", "type", "veth").
+		Run()
+
+	dockerCmd.New().
+		With("exec", fmt.Sprintf("ignite-%s", vmID)).
+		With("ip", "link", "set", "foo", "address", fooAddr).
+		Run()
+
+	// check that the VM has started before trying exec
+	if err := <-vmChans.SpawnFinished; err != nil {
+		t.Fatalf("failed to start a VM: \n%q\n", err)
+	}
+
+	eth1Addr := igniteCmd.New().
+		With("exec", multinetVM).
+		With("cat", "/sys/class/net/eth1/address")
+
+	foundEth1Addr, _ := eth1Addr.Cmd.CombinedOutput()
+	gotEth1Addr := strings.TrimSuffix(string(foundEth1Addr), "\n")
+	assert.Check(t, strings.Contains(gotEth1Addr, fooAddr), fmt.Sprintf("unexpected address found:\n\t(WNT): %q\n\t(GOT): %q", fooAddr, gotEth1Addr))
+
+}
+
+func TestMultipleInterface(t *testing.T) {
+	assert.Assert(t, e2eHome != "", "IGNITE_E2E_HOME should be set")
+
+	igniteCmd := util.NewCommand(t, igniteBin)
+	dockerCmd := util.NewCommand(t, runtime.RuntimeDocker.String())
+
+	vmChans, vmID := startAsyncVM(t, []string{"foo", "bar"})
+
+	// Clean-up the following VM.
+	defer igniteCmd.New().
+		With("rm", "-f", multinetVM).
+		Run()
+
+	fooAddr := "aa:ca:e9:12:34:56"
+	dockerCmd.New().
+		With("exec", fmt.Sprintf("ignite-%s", vmID)).
+		With("ip", "link", "add", "foo", "type", "veth").
+		Run()
+
+	dockerCmd.New().
+		With("exec", fmt.Sprintf("ignite-%s", vmID)).
+		With("ip", "link", "set", "foo", "address", fooAddr).
+		Run()
+
+	barAddr := "aa:ca:e9:12:34:78"
+	dockerCmd.New().
+		With("exec", fmt.Sprintf("ignite-%s", vmID)).
+		With("ip", "link", "add", "bar", "type", "veth").
+		Run()
+
+	dockerCmd.New().
+		With("exec", fmt.Sprintf("ignite-%s", vmID)).
+		With("ip", "link", "set", "bar", "address", barAddr).
+		Run()
+
+	// check that the VM has started before trying exec
+	if err := <-vmChans.SpawnFinished; err != nil {
+		t.Fatalf("failed to start a VM: \n%q\n", err)
+	}
+
+	eth1Addr := igniteCmd.New().
+		With("exec", multinetVM).
+		With("cat", "/sys/class/net/eth1/address")
+
+	foundEth1Addr, _ := eth1Addr.Cmd.CombinedOutput()
+	gotEth1Addr := strings.TrimSuffix(string(foundEth1Addr), "\n")
+	assert.Check(t, strings.Contains(gotEth1Addr, barAddr), fmt.Sprintf("unexpected address found:\n\t(WNT): %q\n\t(GOT): %q", barAddr, gotEth1Addr))
+
+	eth2Addr := igniteCmd.New().
+		With("exec", multinetVM).
+		With("cat", "/sys/class/net/eth2/address")
+
+	foundEth2Addr, _ := eth2Addr.Cmd.CombinedOutput()
+	gotEth2Addr := strings.TrimSuffix(string(foundEth2Addr), "\n")
+	assert.Check(t, strings.Contains(gotEth2Addr, fooAddr), fmt.Sprintf("unexpected address found:\n\t(WNT): %q\n\t(GOT): %q", fooAddr, gotEth2Addr))
+
+}
+
+func TestMultipleInterfaceImplicit(t *testing.T) {
+	assert.Assert(t, e2eHome != "", "IGNITE_E2E_HOME should be set")
+
+	igniteCmd := util.NewCommand(t, igniteBin)
+	dockerCmd := util.NewCommand(t, runtime.RuntimeDocker.String())
+
+	vmChans, vmID := startAsyncVM(t, []string{"foo", "bar"})
+
+	// Clean-up the following VM.
+	defer igniteCmd.New().
+		With("rm", "-f", multinetVM).
+		Run()
+
+	fooAddr := "aa:ca:e9:12:34:56"
+	dockerCmd.New().
+		With("exec", fmt.Sprintf("ignite-%s", vmID)).
+		With("ip", "link", "add", "foo", "type", "veth").
+		Run()
+
+	dockerCmd.New().
+		With("exec", fmt.Sprintf("ignite-%s", vmID)).
+		With("ip", "link", "set", "foo", "address", fooAddr).
+		Run()
+
+	barAddr := "aa:ca:e9:12:34:78"
+	dockerCmd.New().
+		With("exec", fmt.Sprintf("ignite-%s", vmID)).
+		With("ip", "link", "add", "bar", "type", "veth").
+		Run()
+
+	dockerCmd.New().
+		With("exec", fmt.Sprintf("ignite-%s", vmID)).
+		With("ip", "link", "set", "bar", "address", barAddr).
+		Run()
+
+	// this interface should never be found inside a VM
+	bazAddr := "aa:ca:e9:12:34:90"
+	dockerCmd.New().
+		With("exec", fmt.Sprintf("ignite-%s", vmID)).
+		With("ip", "link", "add", "baz", "type", "veth").
+		Run()
+
+	dockerCmd.New().
+		With("exec", fmt.Sprintf("ignite-%s", vmID)).
+		With("ip", "link", "set", "baz", "address", bazAddr).
+		Run()
+
+	// check that the VM has started before trying exec
+	if err := <-vmChans.SpawnFinished; err != nil {
+		t.Fatalf("failed to start a VM: \n%q\n", err)
+	}
+
+	eth1Addr := igniteCmd.New().
+		With("exec", multinetVM).
+		With("cat", "/sys/class/net/eth1/address")
+
+	foundEth1Addr, _ := eth1Addr.Cmd.CombinedOutput()
+	gotEth1Addr := strings.TrimSuffix(string(foundEth1Addr), "\n")
+	assert.Check(t, strings.Contains(gotEth1Addr, barAddr), fmt.Sprintf("unexpected address found:\n\t(WNT): %q\n\t(GOT): %q", barAddr, gotEth1Addr))
+
+	eth2Addr := igniteCmd.New().
+		With("exec", multinetVM).
+		With("cat", "/sys/class/net/eth2/address")
+
+	foundEth2Addr, _ := eth2Addr.Cmd.CombinedOutput()
+	gotEth2Addr := strings.TrimSuffix(string(foundEth2Addr), "\n")
+	assert.Check(t, strings.Contains(gotEth2Addr, fooAddr), fmt.Sprintf("unexpected address found:\n\t(WNT): %q\n\t(GOT): %q", fooAddr, gotEth2Addr))
+
+	eth3Addr := igniteCmd.New().
+		With("exec", multinetVM).
+		With("cat", "/sys/class/net/eth3/address")
+
+	_, foundEth3Err := eth3Addr.Cmd.CombinedOutput()
+	assert.Error(t, foundEth3Err, "exit status 1", fmt.Sprintf("unexpected output when looking for eth3 : \n%s", foundEth3Err))
+
+}

pkg/constants/vm.go

@@ -38,4 +38,10 @@ const (
 	// DEFAULT_SANDBOX_IMAGE_NAME is the name of the default sandbox container
 	// image to be used.
 	DEFAULT_SANDBOX_IMAGE_TAG = "dev"
+
+	// IGNITE_INTERFACE_ANNOTATION is the annotation prefix to store a list of extra interfaces
+	IGNITE_INTERFACE_ANNOTATION = "ignite.weave.works/interface/"
+
+	// IGNITE_SANDBOX_ENV_VAR is the annotation prefix to store a list of env variables
+	IGNITE_SANDBOX_ENV_VAR = "ignite.weave.works/sanbox-env/"
 )

pkg/container/dhcp.go

@@ -11,19 +11,13 @@ import (
 	log "github.com/sirupsen/logrus"
 	api "github.com/weaveworks/ignite/pkg/apis/ignite"
 	"github.com/weaveworks/ignite/pkg/constants"
-	"github.com/weaveworks/ignite/pkg/util"
 )
 
 var leaseDuration, _ = time.ParseDuration(constants.DHCP_INFINITE_LEASE) // Infinite lease time
 
 // StartDHCPServers starts multiple DHCP servers for the VM, one per interface
 // It returns the IP addresses that the API object may post in .status, and a potential error
 func StartDHCPServers(vm *api.VM, dhcpIfaces []DHCPInterface) error {
-	// Generate the MAC addresses for the VM's adapters
-	macAddresses := make([]string, 0, len(dhcpIfaces))
-	if err := util.NewMAC(&macAddresses); err != nil {
-		return fmt.Errorf("failed to generate MAC addresses: %v", err)
-	}
 
 	// Fetch the DNS servers given to the container
 	clientConfig, err := dns.ClientConfigFromFile("/etc/resolv.conf")
@@ -36,9 +30,6 @@ func StartDHCPServers(vm *api.VM, dhcpIfaces []DHCPInterface) error {
 		// Set the VM hostname to the VM ID
 		dhcpIface.Hostname = vm.GetUID().String()
 
-		// Set the MAC address filter for the DHCP server
-		dhcpIface.MACFilter = macAddresses[i]
-
 		// Add the DNS servers from the container
 		dhcpIface.SetDNSServers(clientConfig.Servers)
 

pkg/container/firecracker.go

@@ -20,19 +20,9 @@ import (
 )
 
 // ExecuteFirecracker executes the firecracker process using the Go SDK
-func ExecuteFirecracker(vm *api.VM, dhcpIfaces []DHCPInterface) (err error) {
+func ExecuteFirecracker(vm *api.VM, fcIfaces firecracker.NetworkInterfaces) (err error) {
 	drivePath := vm.SnapshotDev()
 
-	networkInterfaces := make([]firecracker.NetworkInterface, 0, len(dhcpIfaces))
-	for _, dhcpIface := range dhcpIfaces {
-		networkInterfaces = append(networkInterfaces, firecracker.NetworkInterface{
-			StaticConfiguration: &firecracker.StaticNetworkConfiguration{
-				MacAddress:  dhcpIface.MACFilter,
-				HostDevName: dhcpIface.VMTAP,
-			},
-		})
-	}
-
 	vCPUCount := int64(vm.Spec.CPUs)
 	memSizeMib := int64(vm.Spec.Memory.MBytes())
 
@@ -67,7 +57,7 @@ func ExecuteFirecracker(vm *api.VM, dhcpIfaces []DHCPInterface) (err error) {
 			IsRootDevice: firecracker.Bool(true),
 			PathOnHost:   &drivePath,
 		}},
-		NetworkInterfaces: networkInterfaces,
+		NetworkInterfaces: fcIfaces,
 		MachineCfg: models.MachineConfiguration{
 			VcpuCount:  &vCPUCount,
 			MemSizeMib: &memSizeMib,