CNI Implementation

client/allocrunner/network_manager_linux.go

@@ -117,6 +117,9 @@ func netModeToIsolationMode(netMode string) drivers.NetIsolationMode {
 	case "driver":
 		return drivers.NetIsolationModeTask
 	default:
+		if strings.HasPrefix(strings.ToLower(netMode), "cni/") {
+			return drivers.NetIsolationModeGroup
+		}
 		return drivers.NetIsolationModeHost
 	}
 }
@@ -129,9 +132,13 @@ func newNetworkConfigurator(log hclog.Logger, alloc *structs.Allocation, config
 		return &hostNetworkConfigurator{}, nil
 	}
 
-	switch strings.ToLower(tg.Networks[0].Mode) {
-	case "bridge":
+	netMode := strings.ToLower(tg.Networks[0].Mode)
+
+	switch {
+	case netMode == "bridge":
 		return newBridgeNetworkConfigurator(log, config.BridgeNetworkName, config.BridgeNetworkAllocSubnet, config.CNIPath)
+	case strings.HasPrefix(netMode, "cni/"):
+		return newCNINetworkConfigurator(log, config.CNIPath, config.CNIInterfacePrefix, config.CNIConfigDir, netMode[4:])
 	default:
 		return &hostNetworkConfigurator{}, nil
 	}

client/allocrunner/networking_bridge_linux.go

@@ -3,27 +3,14 @@ package allocrunner
 import (
 	"context"
 	"fmt"
-	"math/rand"
-	"os"
-	"path/filepath"
-	"time"
 
-	cni "github.com/containerd/go-cni"
 	"github.com/coreos/go-iptables/iptables"
 	hclog "github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hashicorp/nomad/plugins/drivers"
 )
 
 const (
-	// envCNIPath is the environment variable name to use to derive the CNI path
-	// when it is not explicitly set by the client
-	envCNIPath = "CNI_PATH"
-
-	// defaultCNIPath is the CNI path to use when it is not set by the client
-	// and is not set by environment variable
-	defaultCNIPath = "/opt/cni/bin"
-
 	// defaultNomadBridgeName is the name of the bridge to use when not set by
 	// the client
 	defaultNomadBridgeName = "nomad"
@@ -45,33 +32,19 @@ const (
 // shared bridge, configures masquerading for egress traffic and port mapping
 // for ingress
 type bridgeNetworkConfigurator struct {
-	cni         cni.CNI
+	cni         *cniNetworkConfigurator
 	allocSubnet string
 	bridgeName  string
 
-	rand   *rand.Rand
 	logger hclog.Logger
 }
 
 func newBridgeNetworkConfigurator(log hclog.Logger, bridgeName, ipRange, cniPath string) (*bridgeNetworkConfigurator, error) {
 	b := &bridgeNetworkConfigurator{
 		bridgeName:  bridgeName,
 		allocSubnet: ipRange,
-		rand:        rand.New(rand.NewSource(time.Now().Unix())),
 		logger:      log,
 	}
-	if cniPath == "" {
-		if cniPath = os.Getenv(envCNIPath); cniPath == "" {
-			cniPath = defaultCNIPath
-		}
-	}
-
-	c, err := cni.New(cni.WithPluginDir(filepath.SplitList(cniPath)),
-		cni.WithInterfacePrefix(bridgeNetworkAllocIfPrefix))
-	if err != nil {
-		return nil, err
-	}
-	b.cni = c
 
 	if b.bridgeName == "" {
 		b.bridgeName = defaultNomadBridgeName
@@ -81,6 +54,12 @@ func newBridgeNetworkConfigurator(log hclog.Logger, bridgeName, ipRange, cniPath
 		b.allocSubnet = defaultNomadAllocSubnet
 	}
 
+	c, err := newCNINetworkConfiguratorWithConf(log, cniPath, bridgeNetworkAllocIfPrefix, buildNomadBridgeNetConfig(b.bridgeName, b.allocSubnet))
+	if err != nil {
+		return nil, err
+	}
+	b.cni = c
+
 	return b, nil
 }
 
@@ -148,72 +127,16 @@ func (b *bridgeNetworkConfigurator) Setup(ctx context.Context, alloc *structs.Al
 		return fmt.Errorf("failed to initialize table forwarding rules: %v", err)
 	}
 
-	if err := b.ensureCNIInitialized(); err != nil {
-		return err
-	}
-
-	// Depending on the version of bridge cni plugin (< 0.8.4) a known race could occur
-	// where two alloc attempt to create the nomad bridge at the same time, resulting
-	// in one of them to fail. This retry attempts to overcome those erroneous failures.
-	const retry = 3
-	for attempt := 1; ; attempt++ {
-		//TODO eventually returning the IP from the result would be nice to have in the alloc
-		if _, err := b.cni.Setup(ctx, alloc.ID, spec.Path, cni.WithCapabilityPortMap(getPortMapping(alloc))); err != nil {
-			b.logger.Warn("failed to configure bridge network", "err", err, "attempt", attempt)
-			if attempt == retry {
-				return fmt.Errorf("failed to configure bridge network: %v", err)
-			}
-			// Sleep for 1 second + jitter
-			time.Sleep(time.Second + (time.Duration(b.rand.Int63n(1000)) * time.Millisecond))
-			continue
-		}
-		break
-	}
-
-	return nil
-
+	return b.cni.Setup(ctx, alloc, spec)
 }
 
 // Teardown calls the CNI plugins with the delete action
 func (b *bridgeNetworkConfigurator) Teardown(ctx context.Context, alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) error {
-	if err := b.ensureCNIInitialized(); err != nil {
-		return err
-	}
-
-	return b.cni.Remove(ctx, alloc.ID, spec.Path, cni.WithCapabilityPortMap(getPortMapping(alloc)))
-}
-
-func (b *bridgeNetworkConfigurator) ensureCNIInitialized() error {
-	if err := b.cni.Status(); cni.IsCNINotInitialized(err) {
-		return b.cni.Load(cni.WithConfListBytes(b.buildNomadNetConfig()))
-	} else {
-		return err
-	}
-}
-
-// getPortMapping builds a list of portMapping structs that are used as the
-// portmapping capability arguments for the portmap CNI plugin
-func getPortMapping(alloc *structs.Allocation) []cni.PortMapping {
-	ports := []cni.PortMapping{}
-	for _, network := range alloc.AllocatedResources.Shared.Networks {
-		for _, port := range append(network.DynamicPorts, network.ReservedPorts...) {
-			if port.To < 1 {
-				continue
-			}
-			for _, proto := range []string{"tcp", "udp"} {
-				ports = append(ports, cni.PortMapping{
-					HostPort:      int32(port.Value),
-					ContainerPort: int32(port.To),
-					Protocol:      proto,
-				})
-			}
-		}
-	}
-	return ports
+	return b.cni.Teardown(ctx, alloc, spec)
 }
 
-func (b *bridgeNetworkConfigurator) buildNomadNetConfig() []byte {
-	return []byte(fmt.Sprintf(nomadCNIConfigTemplate, b.bridgeName, b.allocSubnet, cniAdminChainName))
+func buildNomadBridgeNetConfig(bridgeName, subnet string) []byte {
+	return []byte(fmt.Sprintf(nomadCNIConfigTemplate, bridgeName, subnet, cniAdminChainName))
 }
 
 const nomadCNIConfigTemplate = `{

client/allocrunner/networking_cni.go

@@ -0,0 +1,182 @@
+package allocrunner
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	cni "github.com/containerd/go-cni"
+	cnilibrary "github.com/containernetworking/cni/libcni"
+	log "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/hashicorp/nomad/plugins/drivers"
+)
+
+const (
+
+	// envCNIPath is the environment variable name to use to derive the CNI path
+	// when it is not explicitly set by the client
+	envCNIPath = "CNI_PATH"
+
+	// defaultCNIPath is the CNI path to use when it is not set by the client
+	// and is not set by environment variable
+	defaultCNIPath = "/opt/cni/bin"
+
+	// defaultCNIInterfacePrefix is the network interface to use if not set in
+	// client config
+	defaultCNIInterfacePrefix = "eth"
+)
+
+type cniNetworkConfigurator struct {
+	cni     cni.CNI
+	cniConf []byte
+
+	rand   *rand.Rand
+	logger log.Logger
+}
+
+func newCNINetworkConfigurator(logger log.Logger, cniPath, cniInterfacePrefix, cniConfDir, networkName string) (*cniNetworkConfigurator, error) {
+	cniConf, err := loadCNIConf(cniConfDir, networkName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load CNI config: %v", err)
+	}
+
+	return newCNINetworkConfiguratorWithConf(logger, cniPath, cniInterfacePrefix, cniConf)
+}
+
+func newCNINetworkConfiguratorWithConf(logger log.Logger, cniPath, cniInterfacePrefix string, cniConf []byte) (*cniNetworkConfigurator, error) {
+	conf := &cniNetworkConfigurator{
+		cniConf: cniConf,
+		rand:    rand.New(rand.NewSource(time.Now().Unix())),
+		logger:  logger,
+	}
+	if cniPath == "" {
+		if cniPath = os.Getenv(envCNIPath); cniPath == "" {
+			cniPath = defaultCNIPath
+		}
+	}
+
+	if cniInterfacePrefix == "" {
+		cniInterfacePrefix = defaultCNIInterfacePrefix
+	}
+
+	c, err := cni.New(cni.WithPluginDir(filepath.SplitList(cniPath)),
+		cni.WithInterfacePrefix(cniInterfacePrefix))
+	if err != nil {
+		return nil, err
+	}
+	conf.cni = c
+
+	return conf, nil
+}
+
+// Setup calls the CNI plugins with the add action
+func (c *cniNetworkConfigurator) Setup(ctx context.Context, alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) error {
+	if err := c.ensureCNIInitialized(); err != nil {
+		return err
+	}
+
+	// Depending on the version of bridge cni plugin used, a known race could occure
+	// where two alloc attempt to create the nomad bridge at the same time, resulting
+	// in one of them to fail. This rety attempts to overcome those erroneous failures.
+	const retry = 3
+	var firstError error
+	for attempt := 1; ; attempt++ {
+		//TODO eventually returning the IP from the result would be nice to have in the alloc
+		if _, err := c.cni.Setup(ctx, alloc.ID, spec.Path, cni.WithCapabilityPortMap(getPortMapping(alloc))); err != nil {
+			c.logger.Warn("failed to configure network", "err", err, "attempt", attempt)
+			switch attempt {
+			case 1:
+				firstError = err
+			case retry:
+				return fmt.Errorf("failed to configure network: %v", firstError)
+			}
+
+			// Sleep for 1 second + jitter
+			time.Sleep(time.Second + (time.Duration(c.rand.Int63n(1000)) * time.Millisecond))
+			continue
+		}
+		break
+	}
+
+	return nil
+
+}
+
+func loadCNIConf(confDir, name string) ([]byte, error) {
+	files, err := cnilibrary.ConfFiles(confDir, []string{".conf", ".conflist", ".json"})
+	switch {
+	case err != nil:
+		return nil, fmt.Errorf("failed to detect CNI config file: %v", err)
+	case len(files) == 0:
+		return nil, fmt.Errorf("no CNI network config found in %s", confDir)
+	}
+
+	// files contains the network config files associated with cni network.
+	// Use lexicographical way as a defined order for network config files.
+	sort.Strings(files)
+	for _, confFile := range files {
+		if strings.HasSuffix(confFile, ".conflist") {
+			confList, err := cnilibrary.ConfListFromFile(confFile)
+			if err != nil {
+				return nil, fmt.Errorf("failed to load CNI config list file %s: %v", confFile, err)
+			}
+			if confList.Name == name {
+				return confList.Bytes, nil
+			}
+		} else {
+			conf, err := cnilibrary.ConfFromFile(confFile)
+			if err != nil {
+				return nil, fmt.Errorf("failed to load CNI config file %s: %v", confFile, err)
+			}
+			if conf.Network.Name == name {
+				return conf.Bytes, nil
+			}
+		}
+	}
+
+	return nil, fmt.Errorf("CNI network config not found for name %q", name)
+}
+
+// Teardown calls the CNI plugins with the delete action
+func (c *cniNetworkConfigurator) Teardown(ctx context.Context, alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) error {
+	if err := c.ensureCNIInitialized(); err != nil {
+		return err
+	}
+
+	return c.cni.Remove(ctx, alloc.ID, spec.Path, cni.WithCapabilityPortMap(getPortMapping(alloc)))
+}
+
+func (c *cniNetworkConfigurator) ensureCNIInitialized() error {
+	if err := c.cni.Status(); cni.IsCNINotInitialized(err) {
+		return c.cni.Load(cni.WithConfListBytes(c.cniConf))
+	} else {
+		return err
+	}
+}
+
+// getPortMapping builds a list of portMapping structs that are used as the
+// portmapping capability arguments for the portmap CNI plugin
+func getPortMapping(alloc *structs.Allocation) []cni.PortMapping {
+	ports := []cni.PortMapping{}
+	for _, network := range alloc.AllocatedResources.Shared.Networks {
+		for _, port := range append(network.DynamicPorts, network.ReservedPorts...) {
+			if port.To < 1 {
+				continue
+			}
+			for _, proto := range []string{"tcp", "udp"} {
+				ports = append(ports, cni.PortMapping{
+					HostPort:      int32(port.Value),
+					ContainerPort: int32(port.To),
+					Protocol:      proto,
+				})
+			}
+		}
+	}
+	return ports
+}