-
Notifications
You must be signed in to change notification settings - Fork 1.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
b112f88
commit 36f4b56
Showing
25 changed files
with
724 additions
and
180 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,182 @@ | ||
| package allocrunner | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "math/rand" | ||
| "os" | ||
| "path/filepath" | ||
| "sort" | ||
| "strings" | ||
| "time" | ||
|
|
||
| cni "github.com/containerd/go-cni" | ||
| cnilibrary "github.com/containernetworking/cni/libcni" | ||
| log "github.com/hashicorp/go-hclog" | ||
| "github.com/hashicorp/nomad/nomad/structs" | ||
| "github.com/hashicorp/nomad/plugins/drivers" | ||
| ) | ||
|
|
||
| const ( | ||
|
|
||
| // envCNIPath is the environment variable name to use to derive the CNI path | ||
| // when it is not explicitly set by the client | ||
| envCNIPath = "CNI_PATH" | ||
|
|
||
| // defaultCNIPath is the CNI path to use when it is not set by the client | ||
| // and is not set by environment variable | ||
| defaultCNIPath = "/opt/cni/bin" | ||
|
|
||
| // defaultCNIInterfacePrefix is the network interface to use if not set in | ||
| // client config | ||
| defaultCNIInterfacePrefix = "eth" | ||
| ) | ||
|
|
||
| type cniNetworkConfigurator struct { | ||
| cni cni.CNI | ||
| cniConf []byte | ||
|
|
||
| rand *rand.Rand | ||
| logger log.Logger | ||
| } | ||
|
|
||
| func newCNINetworkConfigurator(logger log.Logger, cniPath, cniInterfacePrefix, cniConfDir, networkName string) (*cniNetworkConfigurator, error) { | ||
| cniConf, err := loadCNIConf(cniConfDir, networkName) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to load CNI config: %v", err) | ||
| } | ||
|
|
||
| return newCNINetworkConfiguratorWithConf(logger, cniPath, cniInterfacePrefix, cniConf) | ||
| } | ||
|
|
||
| func newCNINetworkConfiguratorWithConf(logger log.Logger, cniPath, cniInterfacePrefix string, cniConf []byte) (*cniNetworkConfigurator, error) { | ||
| conf := &cniNetworkConfigurator{ | ||
| cniConf: cniConf, | ||
| rand: rand.New(rand.NewSource(time.Now().Unix())), | ||
| logger: logger, | ||
| } | ||
| if cniPath == "" { | ||
| if cniPath = os.Getenv(envCNIPath); cniPath == "" { | ||
| cniPath = defaultCNIPath | ||
| } | ||
| } | ||
|
|
||
| if cniInterfacePrefix == "" { | ||
| cniInterfacePrefix = defaultCNIInterfacePrefix | ||
| } | ||
|
|
||
| c, err := cni.New(cni.WithPluginDir(filepath.SplitList(cniPath)), | ||
| cni.WithInterfacePrefix(cniInterfacePrefix)) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| conf.cni = c | ||
|
|
||
| return conf, nil | ||
| } | ||
|
|
||
| // Setup calls the CNI plugins with the add action | ||
| func (c *cniNetworkConfigurator) Setup(ctx context.Context, alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) error { | ||
| if err := c.ensureCNIInitialized(); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| // Depending on the version of bridge cni plugin used, a known race could occure | ||
| // where two alloc attempt to create the nomad bridge at the same time, resulting | ||
| // in one of them to fail. This rety attempts to overcome those erroneous failures. | ||
| const retry = 3 | ||
| var firstError error | ||
| for attempt := 1; ; attempt++ { | ||
| //TODO eventually returning the IP from the result would be nice to have in the alloc | ||
| if _, err := c.cni.Setup(ctx, alloc.ID, spec.Path, cni.WithCapabilityPortMap(getPortMapping(alloc))); err != nil { | ||
| c.logger.Warn("failed to configure network", "err", err, "attempt", attempt) | ||
| switch attempt { | ||
| case 1: | ||
| firstError = err | ||
| case retry: | ||
| return fmt.Errorf("failed to configure network: %v", firstError) | ||
| } | ||
|
|
||
| // Sleep for 1 second + jitter | ||
| time.Sleep(time.Second + (time.Duration(c.rand.Int63n(1000)) * time.Millisecond)) | ||
| continue | ||
| } | ||
| break | ||
| } | ||
|
|
||
| return nil | ||
|
|
||
| } | ||
|
|
||
| func loadCNIConf(confDir, name string) ([]byte, error) { | ||
| files, err := cnilibrary.ConfFiles(confDir, []string{".conf", ".conflist", ".json"}) | ||
| switch { | ||
| case err != nil: | ||
| return nil, fmt.Errorf("failed to detect CNI config file: %v", err) | ||
| case len(files) == 0: | ||
| return nil, fmt.Errorf("no CNI network config found in %s", confDir) | ||
| } | ||
|
|
||
| // files contains the network config files associated with cni network. | ||
| // Use lexicographical way as a defined order for network config files. | ||
| sort.Strings(files) | ||
| for _, confFile := range files { | ||
| if strings.HasSuffix(confFile, ".conflist") { | ||
| confList, err := cnilibrary.ConfListFromFile(confFile) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to load CNI config list file %s: %v", confFile, err) | ||
| } | ||
| if confList.Name == name { | ||
| return confList.Bytes, nil | ||
| } | ||
| } else { | ||
| conf, err := cnilibrary.ConfFromFile(confFile) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to load CNI config file %s: %v", confFile, err) | ||
| } | ||
| if conf.Network.Name == name { | ||
| return conf.Bytes, nil | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return nil, fmt.Errorf("CNI network config not found for name %q", name) | ||
| } | ||
|
|
||
| // Teardown calls the CNI plugins with the delete action | ||
| func (c *cniNetworkConfigurator) Teardown(ctx context.Context, alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) error { | ||
| if err := c.ensureCNIInitialized(); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return c.cni.Remove(ctx, alloc.ID, spec.Path, cni.WithCapabilityPortMap(getPortMapping(alloc))) | ||
| } | ||
|
|
||
| func (c *cniNetworkConfigurator) ensureCNIInitialized() error { | ||
| if err := c.cni.Status(); cni.IsCNINotInitialized(err) { | ||
| return c.cni.Load(cni.WithConfListBytes(c.cniConf)) | ||
| } else { | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| // getPortMapping builds a list of portMapping structs that are used as the | ||
| // portmapping capability arguments for the portmap CNI plugin | ||
| func getPortMapping(alloc *structs.Allocation) []cni.PortMapping { | ||
| ports := []cni.PortMapping{} | ||
| for _, network := range alloc.AllocatedResources.Shared.Networks { | ||
| for _, port := range append(network.DynamicPorts, network.ReservedPorts...) { | ||
| if port.To < 1 { | ||
| continue | ||
| } | ||
| for _, proto := range []string{"tcp", "udp"} { | ||
| ports = append(ports, cni.PortMapping{ | ||
| HostPort: int32(port.Value), | ||
| ContainerPort: int32(port.To), | ||
| Protocol: proto, | ||
| }) | ||
| } | ||
| } | ||
| } | ||
| return ports | ||
| } |
Oops, something went wrong.