diff --git a/api/resources.go b/api/resources.go index cbd24c4bf813..fe5fb521538e 100644 --- a/api/resources.go +++ b/api/resources.go @@ -89,6 +89,12 @@ type Port struct { To int `mapstructure:"to"` } +type DNSConfig struct { + Servers []string `mapstructure:"servers"` + Searches []string `mapstructure:"searches"` + Options []string `mapstructure:"options"` +} + // NetworkResource is used to describe required network // resources of a given task. type NetworkResource struct { @@ -97,6 +103,7 @@ type NetworkResource struct { CIDR string IP string MBits *int + DNS *DNSConfig ReservedPorts []Port DynamicPorts []Port } diff --git a/client/allocrunner/network_manager_linux.go b/client/allocrunner/network_manager_linux.go index 87da239c4337..b6702a7fe399 100644 --- a/client/allocrunner/network_manager_linux.go +++ b/client/allocrunner/network_manager_linux.go @@ -117,6 +117,9 @@ func netModeToIsolationMode(netMode string) drivers.NetIsolationMode { case "driver": return drivers.NetIsolationModeTask default: + if strings.HasPrefix(strings.ToLower(netMode), "cni/") { + return drivers.NetIsolationModeGroup + } return drivers.NetIsolationModeHost } } @@ -129,9 +132,13 @@ func newNetworkConfigurator(log hclog.Logger, alloc *structs.Allocation, config return &hostNetworkConfigurator{}, nil } - switch strings.ToLower(tg.Networks[0].Mode) { - case "bridge": + netMode := strings.ToLower(tg.Networks[0].Mode) + + switch { + case netMode == "bridge": return newBridgeNetworkConfigurator(log, config.BridgeNetworkName, config.BridgeNetworkAllocSubnet, config.CNIPath) + case strings.HasPrefix(netMode, "cni/"): + return newCNINetworkConfigurator(log, config.CNIPath, config.CNIInterfacePrefix, config.CNIConfigDir, netMode[4:]) default: return &hostNetworkConfigurator{}, nil } diff --git a/client/allocrunner/networking_bridge_linux.go b/client/allocrunner/networking_bridge_linux.go index af07ad1d990c..2bd429b8a61a 100644 --- a/client/allocrunner/networking_bridge_linux.go +++ b/client/allocrunner/networking_bridge_linux.go @@ -3,12 +3,7 @@ package allocrunner import ( "context" "fmt" - "math/rand" - "os" - "path/filepath" - "time" - cni "github.com/containerd/go-cni" "github.com/coreos/go-iptables/iptables" hclog "github.com/hashicorp/go-hclog" "github.com/hashicorp/nomad/nomad/structs" @@ -16,14 +11,6 @@ import ( ) const ( - // envCNIPath is the environment variable name to use to derive the CNI path - // when it is not explicitly set by the client - envCNIPath = "CNI_PATH" - - // defaultCNIPath is the CNI path to use when it is not set by the client - // and is not set by environment variable - defaultCNIPath = "/opt/cni/bin" - // defaultNomadBridgeName is the name of the bridge to use when not set by // the client defaultNomadBridgeName = "nomad" @@ -45,11 +32,10 @@ const ( // shared bridge, configures masquerading for egress traffic and port mapping // for ingress type bridgeNetworkConfigurator struct { - cni cni.CNI + cni *cniNetworkConfigurator allocSubnet string bridgeName string - rand *rand.Rand logger hclog.Logger } @@ -57,21 +43,8 @@ func newBridgeNetworkConfigurator(log hclog.Logger, bridgeName, ipRange, cniPath b := &bridgeNetworkConfigurator{ bridgeName: bridgeName, allocSubnet: ipRange, - rand: rand.New(rand.NewSource(time.Now().Unix())), logger: log, } - if cniPath == "" { - if cniPath = os.Getenv(envCNIPath); cniPath == "" { - cniPath = defaultCNIPath - } - } - - c, err := cni.New(cni.WithPluginDir(filepath.SplitList(cniPath)), - cni.WithInterfacePrefix(bridgeNetworkAllocIfPrefix)) - if err != nil { - return nil, err - } - b.cni = c if b.bridgeName == "" { b.bridgeName = defaultNomadBridgeName @@ -81,6 +54,12 @@ func newBridgeNetworkConfigurator(log hclog.Logger, bridgeName, ipRange, cniPath b.allocSubnet = defaultNomadAllocSubnet } + c, err := newCNINetworkConfiguratorWithConf(log, cniPath, bridgeNetworkAllocIfPrefix, buildNomadBridgeNetConfig(b.bridgeName, b.allocSubnet)) + if err != nil { + return nil, err + } + b.cni = c + return b, nil } @@ -148,72 +127,16 @@ func (b *bridgeNetworkConfigurator) Setup(ctx context.Context, alloc *structs.Al return fmt.Errorf("failed to initialize table forwarding rules: %v", err) } - if err := b.ensureCNIInitialized(); err != nil { - return err - } - - // Depending on the version of bridge cni plugin (< 0.8.4) a known race could occur - // where two alloc attempt to create the nomad bridge at the same time, resulting - // in one of them to fail. This retry attempts to overcome those erroneous failures. - const retry = 3 - for attempt := 1; ; attempt++ { - //TODO eventually returning the IP from the result would be nice to have in the alloc - if _, err := b.cni.Setup(ctx, alloc.ID, spec.Path, cni.WithCapabilityPortMap(getPortMapping(alloc))); err != nil { - b.logger.Warn("failed to configure bridge network", "err", err, "attempt", attempt) - if attempt == retry { - return fmt.Errorf("failed to configure bridge network: %v", err) - } - // Sleep for 1 second + jitter - time.Sleep(time.Second + (time.Duration(b.rand.Int63n(1000)) * time.Millisecond)) - continue - } - break - } - - return nil - + return b.cni.Setup(ctx, alloc, spec) } // Teardown calls the CNI plugins with the delete action func (b *bridgeNetworkConfigurator) Teardown(ctx context.Context, alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) error { - if err := b.ensureCNIInitialized(); err != nil { - return err - } - - return b.cni.Remove(ctx, alloc.ID, spec.Path, cni.WithCapabilityPortMap(getPortMapping(alloc))) -} - -func (b *bridgeNetworkConfigurator) ensureCNIInitialized() error { - if err := b.cni.Status(); cni.IsCNINotInitialized(err) { - return b.cni.Load(cni.WithConfListBytes(b.buildNomadNetConfig())) - } else { - return err - } -} - -// getPortMapping builds a list of portMapping structs that are used as the -// portmapping capability arguments for the portmap CNI plugin -func getPortMapping(alloc *structs.Allocation) []cni.PortMapping { - ports := []cni.PortMapping{} - for _, network := range alloc.AllocatedResources.Shared.Networks { - for _, port := range append(network.DynamicPorts, network.ReservedPorts...) { - if port.To < 1 { - continue - } - for _, proto := range []string{"tcp", "udp"} { - ports = append(ports, cni.PortMapping{ - HostPort: int32(port.Value), - ContainerPort: int32(port.To), - Protocol: proto, - }) - } - } - } - return ports + return b.cni.Teardown(ctx, alloc, spec) } -func (b *bridgeNetworkConfigurator) buildNomadNetConfig() []byte { - return []byte(fmt.Sprintf(nomadCNIConfigTemplate, b.bridgeName, b.allocSubnet, cniAdminChainName)) +func buildNomadBridgeNetConfig(bridgeName, subnet string) []byte { + return []byte(fmt.Sprintf(nomadCNIConfigTemplate, bridgeName, subnet, cniAdminChainName)) } const nomadCNIConfigTemplate = `{ diff --git a/client/allocrunner/networking_cni.go b/client/allocrunner/networking_cni.go new file mode 100644 index 000000000000..15cd2745fd49 --- /dev/null +++ b/client/allocrunner/networking_cni.go @@ -0,0 +1,182 @@ +package allocrunner + +import ( + "context" + "fmt" + "math/rand" + "os" + "path/filepath" + "sort" + "strings" + "time" + + cni "github.com/containerd/go-cni" + cnilibrary "github.com/containernetworking/cni/libcni" + log "github.com/hashicorp/go-hclog" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/plugins/drivers" +) + +const ( + + // envCNIPath is the environment variable name to use to derive the CNI path + // when it is not explicitly set by the client + envCNIPath = "CNI_PATH" + + // defaultCNIPath is the CNI path to use when it is not set by the client + // and is not set by environment variable + defaultCNIPath = "/opt/cni/bin" + + // defaultCNIInterfacePrefix is the network interface to use if not set in + // client config + defaultCNIInterfacePrefix = "eth" +) + +type cniNetworkConfigurator struct { + cni cni.CNI + cniConf []byte + + rand *rand.Rand + logger log.Logger +} + +func newCNINetworkConfigurator(logger log.Logger, cniPath, cniInterfacePrefix, cniConfDir, networkName string) (*cniNetworkConfigurator, error) { + cniConf, err := loadCNIConf(cniConfDir, networkName) + if err != nil { + return nil, fmt.Errorf("failed to load CNI config: %v", err) + } + + return newCNINetworkConfiguratorWithConf(logger, cniPath, cniInterfacePrefix, cniConf) +} + +func newCNINetworkConfiguratorWithConf(logger log.Logger, cniPath, cniInterfacePrefix string, cniConf []byte) (*cniNetworkConfigurator, error) { + conf := &cniNetworkConfigurator{ + cniConf: cniConf, + rand: rand.New(rand.NewSource(time.Now().Unix())), + logger: logger, + } + if cniPath == "" { + if cniPath = os.Getenv(envCNIPath); cniPath == "" { + cniPath = defaultCNIPath + } + } + + if cniInterfacePrefix == "" { + cniInterfacePrefix = defaultCNIInterfacePrefix + } + + c, err := cni.New(cni.WithPluginDir(filepath.SplitList(cniPath)), + cni.WithInterfacePrefix(cniInterfacePrefix)) + if err != nil { + return nil, err + } + conf.cni = c + + return conf, nil +} + +// Setup calls the CNI plugins with the add action +func (c *cniNetworkConfigurator) Setup(ctx context.Context, alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) error { + if err := c.ensureCNIInitialized(); err != nil { + return err + } + + // Depending on the version of bridge cni plugin used, a known race could occure + // where two alloc attempt to create the nomad bridge at the same time, resulting + // in one of them to fail. This rety attempts to overcome those erroneous failures. + const retry = 3 + var firstError error + for attempt := 1; ; attempt++ { + //TODO eventually returning the IP from the result would be nice to have in the alloc + if _, err := c.cni.Setup(ctx, alloc.ID, spec.Path, cni.WithCapabilityPortMap(getPortMapping(alloc))); err != nil { + c.logger.Warn("failed to configure network", "err", err, "attempt", attempt) + switch attempt { + case 1: + firstError = err + case retry: + return fmt.Errorf("failed to configure network: %v", firstError) + } + + // Sleep for 1 second + jitter + time.Sleep(time.Second + (time.Duration(c.rand.Int63n(1000)) * time.Millisecond)) + continue + } + break + } + + return nil + +} + +func loadCNIConf(confDir, name string) ([]byte, error) { + files, err := cnilibrary.ConfFiles(confDir, []string{".conf", ".conflist", ".json"}) + switch { + case err != nil: + return nil, fmt.Errorf("failed to detect CNI config file: %v", err) + case len(files) == 0: + return nil, fmt.Errorf("no CNI network config found in %s", confDir) + } + + // files contains the network config files associated with cni network. + // Use lexicographical way as a defined order for network config files. + sort.Strings(files) + for _, confFile := range files { + if strings.HasSuffix(confFile, ".conflist") { + confList, err := cnilibrary.ConfListFromFile(confFile) + if err != nil { + return nil, fmt.Errorf("failed to load CNI config list file %s: %v", confFile, err) + } + if confList.Name == name { + return confList.Bytes, nil + } + } else { + conf, err := cnilibrary.ConfFromFile(confFile) + if err != nil { + return nil, fmt.Errorf("failed to load CNI config file %s: %v", confFile, err) + } + if conf.Network.Name == name { + return conf.Bytes, nil + } + } + } + + return nil, fmt.Errorf("CNI network config not found for name %q", name) +} + +// Teardown calls the CNI plugins with the delete action +func (c *cniNetworkConfigurator) Teardown(ctx context.Context, alloc *structs.Allocation, spec *drivers.NetworkIsolationSpec) error { + if err := c.ensureCNIInitialized(); err != nil { + return err + } + + return c.cni.Remove(ctx, alloc.ID, spec.Path, cni.WithCapabilityPortMap(getPortMapping(alloc))) +} + +func (c *cniNetworkConfigurator) ensureCNIInitialized() error { + if err := c.cni.Status(); cni.IsCNINotInitialized(err) { + return c.cni.Load(cni.WithConfListBytes(c.cniConf)) + } else { + return err + } +} + +// getPortMapping builds a list of portMapping structs that are used as the +// portmapping capability arguments for the portmap CNI plugin +func getPortMapping(alloc *structs.Allocation) []cni.PortMapping { + ports := []cni.PortMapping{} + for _, network := range alloc.AllocatedResources.Shared.Networks { + for _, port := range append(network.DynamicPorts, network.ReservedPorts...) { + if port.To < 1 { + continue + } + for _, proto := range []string{"tcp", "udp"} { + ports = append(ports, cni.PortMapping{ + HostPort: int32(port.Value), + ContainerPort: int32(port.To), + Protocol: proto, + }) + } + } + } + return ports +} diff --git a/client/allocrunner/taskrunner/task_runner.go b/client/allocrunner/taskrunner/task_runner.go index 2613bf04f7f0..514d0e1ed31c 100644 --- a/client/allocrunner/taskrunner/task_runner.go +++ b/client/allocrunner/taskrunner/task_runner.go @@ -940,6 +940,18 @@ func (tr *TaskRunner) buildTaskConfig() *drivers.TaskConfig { tr.networkIsolationLock.Lock() defer tr.networkIsolationLock.Unlock() + var dns *drivers.DNSConfig + if alloc.AllocatedResources != nil && len(alloc.AllocatedResources.Shared.Networks) > 0 { + allocDNS := alloc.AllocatedResources.Shared.Networks[0].DNS + if allocDNS != nil { + dns = &drivers.DNSConfig{ + Servers: allocDNS.Servers, + Searches: allocDNS.Searches, + Options: allocDNS.Options, + } + } + } + return &drivers.TaskConfig{ ID: fmt.Sprintf("%s/%s/%s", alloc.ID, task.Name, invocationid), Name: task.Name, @@ -963,6 +975,7 @@ func (tr *TaskRunner) buildTaskConfig() *drivers.TaskConfig { StderrPath: tr.logmonHookConfig.stderrFifo, AllocID: tr.allocID, NetworkIsolation: tr.networkIsolationSpec, + DNS: dns, } } diff --git a/client/client.go b/client/client.go index 7e9254955895..b5bd9cdcbb91 100644 --- a/client/client.go +++ b/client/client.go @@ -1436,7 +1436,6 @@ func (c *Client) updateNodeFromFingerprint(response *fingerprint.FingerprintResp // if we still have node changes, merge them if response.Resources != nil { response.Resources.Networks = updateNetworks( - c.config.Node.Resources.Networks, response.Resources.Networks, c.config) if !c.config.Node.Resources.Equals(response.Resources) { @@ -1449,7 +1448,6 @@ func (c *Client) updateNodeFromFingerprint(response *fingerprint.FingerprintResp // if we still have node changes, merge them if response.NodeResources != nil { response.NodeResources.Networks = updateNetworks( - c.config.Node.NodeResources.Networks, response.NodeResources.Networks, c.config) if !c.config.Node.NodeResources.Equals(response.NodeResources) { @@ -1465,33 +1463,40 @@ func (c *Client) updateNodeFromFingerprint(response *fingerprint.FingerprintResp return c.configCopy.Node } -// updateNetworks preserves manually configured network options, but -// applies fingerprint updates -func updateNetworks(ns structs.Networks, up structs.Networks, c *config.Config) structs.Networks { - if c.NetworkInterface == "" { - ns = up - } else { - // If a network device is configured, filter up to contain details for only +// updateNetworks filters and overrides network speed of host networks based +// on configured settings +func updateNetworks(up structs.Networks, c *config.Config) structs.Networks { + if up == nil { + return nil + } + + if c.NetworkInterface != "" { + // For host networks, if a network device is configured filter up to contain details for only // that device upd := []*structs.NetworkResource{} for _, n := range up { - if c.NetworkInterface == n.Device { + switch n.Mode { + case "host": + if c.NetworkInterface == n.Device { + upd = append(upd, n) + } + default: upd = append(upd, n) + } } - // If updates, use them. Otherwise, ns contains the configured interfaces - if len(upd) > 0 { - ns = upd - } + up = upd } - // ns is set, apply the config NetworkSpeed to all + // if set, apply the config NetworkSpeed to networks in host mode if c.NetworkSpeed != 0 { - for _, n := range ns { - n.MBits = c.NetworkSpeed + for _, n := range up { + if n.Mode == "host" { + n.MBits = c.NetworkSpeed + } } } - return ns + return up } // retryIntv calculates a retry interval value given the base diff --git a/client/client_test.go b/client/client_test.go index 95bf51069ade..1afb34422fc6 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -1184,17 +1184,16 @@ func TestClient_UpdateNodeFromFingerprintKeepsConfig(t *testing.T) { client.updateNodeFromFingerprint(&fingerprint.FingerprintResponse{ NodeResources: &structs.NodeResources{ Cpu: structs.NodeCpuResources{CpuShares: 123}, - Networks: []*structs.NetworkResource{{Device: "any-interface"}}, + Networks: []*structs.NetworkResource{{Mode: "host", Device: "any-interface"}}, }, Resources: &structs.Resources{ - CPU: 80, - Networks: []*structs.NetworkResource{{Device: "any-interface"}}, + CPU: 80, }, }) - assert.Equal(t, int64(123), client.config.Node.NodeResources.Cpu.CpuShares) - assert.Equal(t, "any-interface", client.config.Node.NodeResources.Networks[0].Device) - assert.Equal(t, 80, client.config.Node.Resources.CPU) - assert.Equal(t, "any-interface", client.config.Node.Resources.Networks[0].Device) + idx := len(client.config.Node.NodeResources.Networks) - 1 + require.Equal(t, int64(123), client.config.Node.NodeResources.Cpu.CpuShares) + require.Equal(t, "any-interface", client.config.Node.NodeResources.Networks[idx].Device) + require.Equal(t, 80, client.config.Node.Resources.CPU) // lookup an interface. client.Node starts with a hardcoded value, eth0, // and is only updated async through fingerprinter. @@ -1210,48 +1209,43 @@ func TestClient_UpdateNodeFromFingerprintKeepsConfig(t *testing.T) { client, cleanup = TestClient(t, func(c *config.Config) { c.NetworkInterface = dev c.Node.Name = name + c.Options["fingerprint.blacklist"] = "network" // Node is already a mock.Node, with a device c.Node.NodeResources.Networks[0].Device = dev - c.Node.Resources.Networks = c.Node.NodeResources.Networks }) defer cleanup() client.updateNodeFromFingerprint(&fingerprint.FingerprintResponse{ NodeResources: &structs.NodeResources{ Cpu: structs.NodeCpuResources{CpuShares: 123}, Networks: []*structs.NetworkResource{ - {Device: "any-interface", MBits: 20}, - {Device: dev, MBits: 20}, + {Mode: "host", Device: "any-interface", MBits: 20}, }, }, - Resources: &structs.Resources{ - CPU: 80, - Networks: []*structs.NetworkResource{{Device: "any-interface"}}, - }, }) - assert.Equal(t, int64(123), client.config.Node.NodeResources.Cpu.CpuShares) + require.Equal(t, int64(123), client.config.Node.NodeResources.Cpu.CpuShares) // only the configured device is kept - assert.Equal(t, 1, len(client.config.Node.NodeResources.Networks)) - assert.Equal(t, dev, client.config.Node.NodeResources.Networks[0].Device) - // network speed updates to the configured network are kept - assert.Equal(t, 20, client.config.Node.NodeResources.Networks[0].MBits) - assert.Equal(t, 80, client.config.Node.Resources.CPU) - assert.Equal(t, dev, client.config.Node.Resources.Networks[0].Device) + require.Equal(t, 2, len(client.config.Node.NodeResources.Networks)) + require.Equal(t, dev, client.config.Node.NodeResources.Networks[0].Device) + require.Equal(t, "bridge", client.config.Node.NodeResources.Networks[1].Mode) // Network speed is applied to all NetworkResources client.config.NetworkInterface = "" client.config.NetworkSpeed = 100 client.updateNodeFromFingerprint(&fingerprint.FingerprintResponse{ NodeResources: &structs.NodeResources{ - Cpu: structs.NodeCpuResources{CpuShares: 123}, - Networks: []*structs.NetworkResource{{Device: "any-interface", MBits: 20}}, + Cpu: structs.NodeCpuResources{CpuShares: 123}, + Networks: []*structs.NetworkResource{ + {Mode: "host", Device: "any-interface", MBits: 20}, + }, }, Resources: &structs.Resources{ - CPU: 80, - Networks: []*structs.NetworkResource{{Device: "any-interface"}}, + CPU: 80, }, }) - assert.Equal(t, "any-interface", client.config.Node.NodeResources.Networks[0].Device) - assert.Equal(t, 100, client.config.Node.NodeResources.Networks[0].MBits) + assert.Equal(t, 3, len(client.config.Node.NodeResources.Networks)) + assert.Equal(t, "any-interface", client.config.Node.NodeResources.Networks[2].Device) + assert.Equal(t, 100, client.config.Node.NodeResources.Networks[2].MBits) + assert.Equal(t, 0, client.config.Node.NodeResources.Networks[1].MBits) } // Support multiple IP addresses (ipv4 vs. 6, e.g.) on the configured network interface @@ -1269,7 +1263,7 @@ func Test_UpdateNodeFromFingerprintMultiIP(t *testing.T) { // Client without network configured updates to match fingerprint client, cleanup := TestClient(t, func(c *config.Config) { c.NetworkInterface = dev - c.Node.NodeResources.Networks[0].Device = dev + c.Options["fingerprint.blacklist"] = "network,cni,bridge" c.Node.Resources.Networks = c.Node.NodeResources.Networks }) defer cleanup() @@ -1284,12 +1278,13 @@ func Test_UpdateNodeFromFingerprintMultiIP(t *testing.T) { }, }) - two := structs.Networks{ + nets := structs.Networks{ + mock.Node().NodeResources.Networks[0], {Device: dev, IP: "127.0.0.1"}, {Device: dev, IP: "::1"}, } - require.Equal(t, two, client.config.Node.NodeResources.Networks) + require.Equal(t, nets, client.config.Node.NodeResources.Networks) } func TestClient_computeAllocatedDeviceStats(t *testing.T) { @@ -1480,6 +1475,9 @@ func TestClient_getAllocatedResources(t *testing.T) { result := client.getAllocatedResources(client.config.Node) + // Ignore comparing networks for now + result.Flattened.Networks = nil + expected := structs.ComparableResources{ Flattened: structs.AllocatedTaskResources{ Cpu: structs.AllocatedCpuResources{ diff --git a/client/config/config.go b/client/config/config.go index d1de0d2eaa82..a6df2fac2d18 100644 --- a/client/config/config.go +++ b/client/config/config.go @@ -234,6 +234,15 @@ type Config struct { // be specified with colon delimited CNIPath string + // CNIConfigDir is the directory where CNI network configuration is located. The + // client will use this path when fingerprinting CNI networks. + CNIConfigDir string + + // CNIInterfacePrefix is the prefix to use when creating CNI network interfaces. This + // defaults to 'eth', therefore the first interface created by CNI inside the alloc + // network will be 'eth0'. + CNIInterfacePrefix string + // BridgeNetworkName is the name to use for the bridge created in bridge // networking mode. This defaults to 'nomad' if not set BridgeNetworkName string @@ -301,6 +310,9 @@ func DefaultConfig() *Config { }, BackwardsCompatibleMetrics: false, RPCHoldTimeout: 5 * time.Second, + CNIPath: "/opt/cni/bin", + CNIConfigDir: "/opt/cni/config", + CNIInterfacePrefix: "eth", } } diff --git a/client/fingerprint/bridge.go b/client/fingerprint/bridge.go new file mode 100644 index 000000000000..163e8cbea191 --- /dev/null +++ b/client/fingerprint/bridge.go @@ -0,0 +1,12 @@ +package fingerprint + +import log "github.com/hashicorp/go-hclog" + +type BridgeFingerprint struct { + logger log.Logger + StaticFingerprinter +} + +func NewBridgeFingerprint(logger log.Logger) Fingerprint { + return &BridgeFingerprint{logger: logger} +} diff --git a/client/fingerprint/bridge_default.go b/client/fingerprint/bridge_default.go new file mode 100644 index 000000000000..bf5445227301 --- /dev/null +++ b/client/fingerprint/bridge_default.go @@ -0,0 +1,5 @@ +// +build !linux + +package fingerprint + +func (f *BridgeFingerprint) Fingerprint(*FingerprintRequest, *FingerprintResponse) error { return nil } diff --git a/client/fingerprint/bridge_linux.go b/client/fingerprint/bridge_linux.go new file mode 100644 index 000000000000..231d669ecfe9 --- /dev/null +++ b/client/fingerprint/bridge_linux.go @@ -0,0 +1,49 @@ +package fingerprint + +import ( + "bufio" + "fmt" + "os" + "regexp" + + "github.com/hashicorp/nomad/nomad/structs" +) + +const bridgeKernelModuleName = "bridge" + +func (f *BridgeFingerprint) Fingerprint(req *FingerprintRequest, resp *FingerprintResponse) error { + if err := f.checkKMod(bridgeKernelModuleName); err != nil { + f.logger.Warn("failed to detect bridge kernel module, bridge network mode disabled", "error", err) + return nil + } + + resp.NodeResources = &structs.NodeResources{ + Networks: []*structs.NetworkResource{ + { + Mode: "bridge", + }, + }, + } + resp.Detected = true + return nil +} + +func (f *BridgeFingerprint) checkKMod(mod string) error { + file, err := os.Open("/proc/modules") + if err != nil { + return fmt.Errorf("could not read /proc/modules: %v", err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + pattern := fmt.Sprintf("%s\\s+.*$", mod) + for scanner.Scan() { + if matched, err := regexp.MatchString(pattern, scanner.Text()); matched { + return nil + } else if err != nil { + return fmt.Errorf("could not parse /proc/modules: %v", err) + } + } + + return fmt.Errorf("could not detect kernel module %s", mod) +} diff --git a/client/fingerprint/bridge_linux_test.go b/client/fingerprint/bridge_linux_test.go new file mode 100644 index 000000000000..4e55c92dcee4 --- /dev/null +++ b/client/fingerprint/bridge_linux_test.go @@ -0,0 +1,14 @@ +package fingerprint + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBridgeFingerprint_checkKMod(t *testing.T) { + require := require.New(t) + f := &BridgeFingerprint{} + require.NoError(f.checkKMod("ip_tables")) + require.Error(f.checkKMod("nonexistentmodule")) +} diff --git a/client/fingerprint/cni.go b/client/fingerprint/cni.go new file mode 100644 index 000000000000..9080733db592 --- /dev/null +++ b/client/fingerprint/cni.go @@ -0,0 +1,77 @@ +package fingerprint + +import ( + "fmt" + "os" + "strings" + + "github.com/containernetworking/cni/libcni" + log "github.com/hashicorp/go-hclog" + "github.com/hashicorp/nomad/nomad/structs" +) + +type CNIFingerprint struct { + StaticFingerprinter + logger log.Logger +} + +func NewCNIFingerprint(logger log.Logger) Fingerprint { + return &CNIFingerprint{logger: logger} +} + +func (f *CNIFingerprint) Fingerprint(req *FingerprintRequest, resp *FingerprintResponse) error { + confDir := req.Config.CNIConfigDir + networks := map[string]struct{}{} + if _, err := os.Stat(confDir); os.IsNotExist(err) { + f.logger.Debug("CNI config dir is not set or does not exist, skipping", "cni_config_dir", confDir) + resp.Detected = false + return nil + } + + files, err := libcni.ConfFiles(confDir, []string{".conf", ".conflist", ".json"}) + if err != nil { + return fmt.Errorf("failed to detect CNI conf files: %v", err) + } + + for _, confFile := range files { + if strings.HasSuffix(confFile, ".conflist") { + confList, err := libcni.ConfListFromFile(confFile) + if err != nil { + return fmt.Errorf("failed to load CNI config list file %s: %v", confFile, err) + } + if _, ok := networks[confList.Name]; ok { + f.logger.Warn("duplicate CNI config names found, ignoring file", "name", confList.Name, "file", confFile) + continue + } + networks[confList.Name] = struct{}{} + } else { + conf, err := libcni.ConfFromFile(confFile) + if err != nil { + return fmt.Errorf("failed to load CNI config file %s: %v", confFile, err) + } + if _, ok := networks[conf.Network.Name]; ok { + f.logger.Warn("duplicate CNI config names found, ignoring file", "name", conf.Network.Name, "file", confFile) + continue + } + networks[conf.Network.Name] = struct{}{} + } + } + + var nodeNetworks structs.Networks + + for name := range networks { + nodeNetworks = append(nodeNetworks, &structs.NetworkResource{ + Mode: fmt.Sprintf("cni/%s", name), + }) + f.logger.Debug("detected CNI network", "name", name) + } + + resp.NodeResources = &structs.NodeResources{ + Networks: nodeNetworks, + } + + resp.Detected = true + return nil +} + +func (f *CNIFingerprint) Reload() {} diff --git a/client/fingerprint/cni_test.go b/client/fingerprint/cni_test.go new file mode 100644 index 000000000000..3fd125b7d4f6 --- /dev/null +++ b/client/fingerprint/cni_test.go @@ -0,0 +1,85 @@ +package fingerprint + +import ( + "testing" + + "github.com/hashicorp/nomad/client/config" + "github.com/hashicorp/nomad/helper/testlog" + "github.com/hashicorp/nomad/nomad/structs" + "github.com/stretchr/testify/require" +) + +// Test that CNI fingerprinter is reloadable +var _ ReloadableFingerprint = &CNIFingerprint{} + +func TestCNIFingerprint(t *testing.T) { + cases := []struct { + name string + req *FingerprintRequest + exp *FingerprintResponse + err bool + errMatch string + }{ + { + name: "cni config dir not set", + req: &FingerprintRequest{ + Config: &config.Config{}, + }, + exp: &FingerprintResponse{ + Detected: false, + }, + }, + { + name: "cni config dir non-existent", + req: &FingerprintRequest{ + Config: &config.Config{ + CNIConfigDir: "text_fixtures/cni_nonexistent", + }, + }, + exp: &FingerprintResponse{ + Detected: false, + }, + }, + { + name: "two networks, no errors", + req: &FingerprintRequest{ + Config: &config.Config{ + CNIConfigDir: "test_fixtures/cni", + }, + }, + exp: &FingerprintResponse{ + NodeResources: &structs.NodeResources{ + Networks: []*structs.NetworkResource{ + { + Mode: "cni/net1", + }, + { + Mode: "cni/net2", + }, + }, + }, + Detected: true, + }, + err: false, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + r := require.New(t) + fp := NewCNIFingerprint(testlog.HCLogger(t)) + resp := &FingerprintResponse{} + err := fp.Fingerprint(c.req, resp) + if c.err { + r.Error(err) + r.Contains(err.Error(), c.errMatch) + } else { + r.NoError(err) + r.Equal(c.exp.Detected, resp.Detected) + if resp.NodeResources != nil || c.exp.NodeResources != nil { + r.ElementsMatch(c.exp.NodeResources.Networks, resp.NodeResources.Networks) + } + } + }) + } +} diff --git a/client/fingerprint/env_aws.go b/client/fingerprint/env_aws.go index 0b2466fd3bb3..87becff9e9aa 100644 --- a/client/fingerprint/env_aws.go +++ b/client/fingerprint/env_aws.go @@ -485,6 +485,7 @@ func (f *EnvAWSFingerprint) Fingerprint(request *FingerprintRequest, response *F nodeResources = new(structs.NodeResources) nodeResources.Networks = []*structs.NetworkResource{ { + Mode: "host", Device: "eth0", IP: val, CIDR: val + "/32", diff --git a/client/fingerprint/fingerprint.go b/client/fingerprint/fingerprint.go index f7d0e2279b70..d896b48e2240 100644 --- a/client/fingerprint/fingerprint.go +++ b/client/fingerprint/fingerprint.go @@ -31,6 +31,7 @@ var ( hostFingerprinters = map[string]Factory{ "arch": NewArchFingerprint, "consul": NewConsulFingerprint, + "cni": NewCNIFingerprint, "cpu": NewCPUFingerprint, "host": NewHostFingerprint, "memory": NewMemoryFingerprint, @@ -115,6 +116,13 @@ type Fingerprint interface { Periodic() (bool, time.Duration) } +// ReloadableFingerprint can be implemented if the fingerprinter needs to be run during client reload. +// If implemented, the client will call Reload during client reload then immediately Fingerprint +type ReloadableFingerprint interface { + Fingerprint + Reload() +} + // StaticFingerprinter can be embedded in a struct that has a Fingerprint method // to make it non-periodic. type StaticFingerprinter struct{} diff --git a/client/fingerprint/fingerprint_linux.go b/client/fingerprint/fingerprint_linux.go index f52669a7f18b..10c9768b29f4 100644 --- a/client/fingerprint/fingerprint_linux.go +++ b/client/fingerprint/fingerprint_linux.go @@ -2,4 +2,5 @@ package fingerprint func initPlatformFingerprints(fps map[string]Factory) { fps["cgroup"] = NewCGroupFingerprint + fps["bridge"] = NewBridgeFingerprint } diff --git a/client/fingerprint/network.go b/client/fingerprint/network.go index 9cc4fcc658f7..73b8fa5098c4 100644 --- a/client/fingerprint/network.go +++ b/client/fingerprint/network.go @@ -132,6 +132,7 @@ func (f *NetworkFingerprint) createNetworkResources(throughput int, intf *net.In for _, addr := range addrs { // Create a new network resource newNetwork := &structs.NetworkResource{ + Mode: "host", Device: intf.Name, MBits: throughput, } diff --git a/client/fingerprint/test_fixtures/cni/net1.conf b/client/fingerprint/test_fixtures/cni/net1.conf new file mode 100644 index 000000000000..ea5dddb63d8e --- /dev/null +++ b/client/fingerprint/test_fixtures/cni/net1.conf @@ -0,0 +1,17 @@ +{ + "cniVersion": "0.2.0", + "name": "net1", + "type": "bridge", + "bridge": "cni0", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "10.22.0.0/16", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ] + } +} diff --git a/client/fingerprint/test_fixtures/cni/net2.conflist b/client/fingerprint/test_fixtures/cni/net2.conflist new file mode 100644 index 000000000000..cec6c67efa95 --- /dev/null +++ b/client/fingerprint/test_fixtures/cni/net2.conflist @@ -0,0 +1,25 @@ +{ + "cniVersion": "0.3.1", + "name": "net2", + "plugins": [ + { + "type": "ptp", + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "172.16.30.0/24", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ] + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + } + ] +} diff --git a/client/fingerprint_manager.go b/client/fingerprint_manager.go index 84efed5bcb38..e8993526ffaf 100644 --- a/client/fingerprint_manager.go +++ b/client/fingerprint_manager.go @@ -24,6 +24,8 @@ type FingerprintManager struct { // associated node updateNodeAttributes func(*fingerprint.FingerprintResponse) *structs.Node + reloadableFps map[string]fingerprint.ReloadableFingerprint + logger log.Logger } @@ -44,6 +46,7 @@ func NewFingerprintManager( node: node, shutdownCh: shutdownCh, logger: logger.Named("fingerprint_mgr"), + reloadableFps: make(map[string]fingerprint.ReloadableFingerprint), } } @@ -103,6 +106,17 @@ func (fp *FingerprintManager) Run() error { return nil } +// Reload will reload any registered ReloadableFingerprinters and immediately call Fingerprint +func (fm *FingerprintManager) Reload() { + for name, fp := range fm.reloadableFps { + fm.logger.Info("reloading fingerprinter", "fingerprinter", name) + fp.Reload() + if _, err := fm.fingerprint(name, fp); err != nil { + fm.logger.Warn("error fingerprinting after reload", "fingerprinter", name, "error", err) + } + } +} + // setupFingerprints is used to fingerprint the node to see if these attributes are // supported func (fm *FingerprintManager) setupFingerprinters(fingerprints []string) error { @@ -130,6 +144,10 @@ func (fm *FingerprintManager) setupFingerprinters(fingerprints []string) error { if p { go fm.runFingerprint(f, period, name) } + + if rfp, ok := f.(fingerprint.ReloadableFingerprint); ok { + fm.reloadableFps[name] = rfp + } } fm.logger.Debug("detected fingerprints", "node_attrs", appliedFingerprints) diff --git a/command/agent/job_endpoint.go b/command/agent/job_endpoint.go index b2c6db3f9c61..9c282df2bce6 100644 --- a/command/agent/job_endpoint.go +++ b/command/agent/job_endpoint.go @@ -1132,6 +1132,14 @@ func ApiNetworkResourceToStructs(in []*api.NetworkResource) []*structs.NetworkRe MBits: *nw.MBits, } + if nw.DNS != nil { + out[i].DNS = &structs.DNSConfig{ + Servers: nw.DNS.Servers, + Searches: nw.DNS.Searches, + Options: nw.DNS.Options, + } + } + if l := len(nw.DynamicPorts); l != 0 { out[i].DynamicPorts = make([]structs.Port, l) for j, dp := range nw.DynamicPorts { diff --git a/command/assets/connect.nomad b/command/assets/connect.nomad index 59b6ca64354e..c7f6c6404ebb 100644 --- a/command/assets/connect.nomad +++ b/command/assets/connect.nomad @@ -256,6 +256,12 @@ job "countdash" { # port "http" { # to = "8080" # } + + # The "dns" stanza allows operators to override the DNS configuration + # inherited by the host client. + # dns { + # servers = ["1.1.1.1"] + # } } # The "service" stanza enables Consul Connect. service { diff --git a/drivers/docker/driver.go b/drivers/docker/driver.go index 97e60967be47..59cb459198d2 100644 --- a/drivers/docker/driver.go +++ b/drivers/docker/driver.go @@ -915,6 +915,19 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T hostConfig.ShmSize = driverConfig.ShmSize } + // setup Nomad DNS options, these are overridden by docker driver specific options + if task.DNS != nil { + hostConfig.DNS = task.DNS.Servers + hostConfig.DNSSearch = task.DNS.Searches + hostConfig.DNSOptions = task.DNS.Options + } + + if len(driverConfig.DNSSearchDomains) > 0 { + hostConfig.DNSSearch = driverConfig.DNSSearchDomains + } + if len(driverConfig.DNSOptions) > 0 { + hostConfig.DNSOptions = driverConfig.DNSOptions + } // set DNS servers for _, ip := range driverConfig.DNSServers { if net.ParseIP(ip) != nil { @@ -978,9 +991,6 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T hostConfig.Mounts = append(hostConfig.Mounts, hm) } - // set DNS search domains and extra hosts - hostConfig.DNSSearch = driverConfig.DNSSearchDomains - hostConfig.DNSOptions = driverConfig.DNSOptions hostConfig.ExtraHosts = driverConfig.ExtraHosts hostConfig.IpcMode = driverConfig.IPCMode diff --git a/drivers/docker/driver_unix_test.go b/drivers/docker/driver_unix_test.go index 39beed99429e..27243193f890 100644 --- a/drivers/docker/driver_unix_test.go +++ b/drivers/docker/driver_unix_test.go @@ -810,3 +810,59 @@ func TestDocker_ExecTaskStreaming(t *testing.T) { dtestutil.ExecTaskStreamingConformanceTests(t, d, task.ID) } + +// Tests that a given DNSConfig properly configures dns +func Test_dnsConfig(t *testing.T) { + if !tu.IsCI() { + t.Parallel() + } + testutil.DockerCompatible(t) + require := require.New(t) + harness := dockerDriverHarness(t, nil) + defer harness.Kill() + + cases := []struct { + name string + cfg *drivers.DNSConfig + }{ + { + name: "nil DNSConfig", + }, + { + name: "basic", + cfg: &drivers.DNSConfig{ + Servers: []string{"1.1.1.1", "1.0.0.1"}, + }, + }, + { + name: "full", + cfg: &drivers.DNSConfig{ + Servers: []string{"1.1.1.1", "1.0.0.1"}, + Searches: []string{"local.test", "node.consul"}, + Options: []string{"ndots:2", "edns0"}, + }, + }, + } + + for _, c := range cases { + taskCfg := newTaskConfig("", []string{"/bin/sleep", "1000"}) + task := &drivers.TaskConfig{ + ID: uuid.Generate(), + Name: "nc-demo", + AllocID: uuid.Generate(), + Resources: basicResources, + DNS: c.cfg, + } + require.NoError(task.EncodeConcreteDriverConfig(&taskCfg)) + + cleanup := harness.MkAllocDir(task, false) + defer cleanup() + + _, _, err := harness.StartTask(task) + require.NoError(err) + defer harness.DestroyTask(task.ID, true) + + dtestutil.TestTaskDNSConfig(t, harness, task.ID, c.cfg) + } + +} diff --git a/drivers/exec/driver.go b/drivers/exec/driver.go index 22094216dac3..6a82f7779767 100644 --- a/drivers/exec/driver.go +++ b/drivers/exec/driver.go @@ -14,6 +14,7 @@ import ( "github.com/hashicorp/nomad/client/fingerprint" "github.com/hashicorp/nomad/drivers/shared/eventer" "github.com/hashicorp/nomad/drivers/shared/executor" + "github.com/hashicorp/nomad/drivers/shared/resolvconf" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/pluginutils/loader" "github.com/hashicorp/nomad/plugins/base" @@ -360,6 +361,14 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive user = "nobody" } + if cfg.DNS != nil { + dnsMount, err := resolvconf.GenerateDNSMount(cfg.TaskDir().Dir, cfg.DNS) + if err != nil { + return nil, nil, fmt.Errorf("failed to build mount for resolv.conf: %v", err) + } + cfg.Mounts = append(cfg.Mounts, dnsMount) + } + execCmd := &executor.ExecCommand{ Cmd: driverConfig.Command, Args: driverConfig.Args, diff --git a/drivers/exec/driver_unix_test.go b/drivers/exec/driver_unix_test.go index 5ff063745c9c..cf1e838a3afe 100644 --- a/drivers/exec/driver_unix_test.go +++ b/drivers/exec/driver_unix_test.go @@ -113,3 +113,65 @@ func TestExec_ExecTaskStreaming(t *testing.T) { dtestutil.ExecTaskStreamingConformanceTests(t, harness, task.ID) } + +// Tests that a given DNSConfig properly configures dns +func TestExec_dnsConfig(t *testing.T) { + t.Parallel() + ctestutils.RequireRoot(t) + ctestutils.ExecCompatible(t) + require := require.New(t) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + d := NewExecDriver(ctx, testlog.HCLogger(t)) + harness := dtestutil.NewDriverHarness(t, d) + defer harness.Kill() + + cases := []struct { + name string + cfg *drivers.DNSConfig + }{ + { + name: "nil DNSConfig", + }, + { + name: "basic", + cfg: &drivers.DNSConfig{ + Servers: []string{"1.1.1.1", "1.0.0.1"}, + }, + }, + { + name: "full", + cfg: &drivers.DNSConfig{ + Servers: []string{"1.1.1.1", "1.0.0.1"}, + Searches: []string{"local.test", "node.consul"}, + Options: []string{"ndots:2", "edns0"}, + }, + }, + } + + for _, c := range cases { + task := &drivers.TaskConfig{ + ID: uuid.Generate(), + Name: "sleep", + DNS: c.cfg, + } + + cleanup := harness.MkAllocDir(task, false) + defer cleanup() + + tc := &TaskConfig{ + Command: "/bin/sleep", + Args: []string{"9000"}, + } + require.NoError(task.EncodeConcreteDriverConfig(&tc)) + + _, _, err := harness.StartTask(task) + require.NoError(err) + defer d.DestroyTask(task.ID, true) + + dtestutil.TestTaskDNSConfig(t, harness, task.ID, c.cfg) + } + +} diff --git a/drivers/java/driver.go b/drivers/java/driver.go index f8aa408f6bca..220935046941 100644 --- a/drivers/java/driver.go +++ b/drivers/java/driver.go @@ -14,6 +14,7 @@ import ( "github.com/hashicorp/nomad/client/fingerprint" "github.com/hashicorp/nomad/drivers/shared/eventer" "github.com/hashicorp/nomad/drivers/shared/executor" + "github.com/hashicorp/nomad/drivers/shared/resolvconf" "github.com/hashicorp/nomad/helper/pluginutils/loader" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" @@ -340,6 +341,14 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive user = "nobody" } + if cfg.DNS != nil { + dnsMount, err := resolvconf.GenerateDNSMount(cfg.TaskDir().Dir, cfg.DNS) + if err != nil { + return nil, nil, fmt.Errorf("failed to build mount for resolv.conf: %v", err) + } + cfg.Mounts = append(cfg.Mounts, dnsMount) + } + execCmd := &executor.ExecCommand{ Cmd: absPath, Args: args, diff --git a/drivers/java/driver_test.go b/drivers/java/driver_test.go index 4aed991f2529..b4c2c9e4072f 100644 --- a/drivers/java/driver_test.go +++ b/drivers/java/driver_test.go @@ -360,3 +360,59 @@ config { require.EqualValues(t, expected, tc) } + +// Tests that a given DNSConfig properly configures dns +func Test_dnsConfig(t *testing.T) { + t.Parallel() + ctestutil.RequireRoot(t) + javaCompatible(t) + require := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + d := NewDriver(ctx, testlog.HCLogger(t)) + harness := dtestutil.NewDriverHarness(t, d) + defer harness.Kill() + + cases := []struct { + name string + cfg *drivers.DNSConfig + }{ + { + name: "nil DNSConfig", + }, + { + name: "basic", + cfg: &drivers.DNSConfig{ + Servers: []string{"1.1.1.1", "1.0.0.1"}, + }, + }, + { + name: "full", + cfg: &drivers.DNSConfig{ + Servers: []string{"1.1.1.1", "1.0.0.1"}, + Searches: []string{"local.test", "node.consul"}, + Options: []string{"ndots:2", "edns0"}, + }, + }, + } + + for _, c := range cases { + tc := &TaskConfig{ + Class: "Hello", + Args: []string{"900"}, + } + task := basicTask(t, "demo-app", tc) + task.DNS = c.cfg + + cleanup := harness.MkAllocDir(task, false) + defer cleanup() + + _, _, err := harness.StartTask(task) + require.NoError(err) + defer d.DestroyTask(task.ID, true) + + dtestutil.TestTaskDNSConfig(t, harness, task.ID, c.cfg) + } + +} diff --git a/drivers/qemu/driver.go b/drivers/qemu/driver.go index 24805d0b2b98..b60be22a7149 100644 --- a/drivers/qemu/driver.go +++ b/drivers/qemu/driver.go @@ -341,6 +341,17 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive "-nographic", } + var netdevArgs []string + if cfg.DNS != nil { + if len(cfg.DNS.Servers) > 0 { + netdevArgs = append(netdevArgs, "dns="+cfg.DNS.Servers[0]) + } + + for _, s := range cfg.DNS.Searches { + netdevArgs = append(netdevArgs, "dnssearch="+s) + } + } + var monitorPath string if driverConfig.GracefulShutdown { if runtime.GOOS == "windows" { @@ -379,7 +390,6 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive // Loop through the port map and construct the hostfwd string, to map // reserved ports to the ports listenting in the VM // Ex: hostfwd=tcp::22000-:22,hostfwd=tcp::80-:8080 - var forwarding []string taskPorts := cfg.Resources.NomadResources.Networks[0].PortLabels() for label, guest := range driverConfig.PortMap { host, ok := taskPorts[label] @@ -388,14 +398,14 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive } for _, p := range protocols { - forwarding = append(forwarding, fmt.Sprintf("hostfwd=%s::%d-:%d", p, host, guest)) + netdevArgs = append(netdevArgs, fmt.Sprintf("hostfwd=%s::%d-:%d", p, host, guest)) } } - if len(forwarding) != 0 { + if len(netdevArgs) != 0 { args = append(args, "-netdev", - fmt.Sprintf("user,id=user.0,%s", strings.Join(forwarding, ",")), + fmt.Sprintf("user,id=user.0,%s", strings.Join(netdevArgs, ",")), "-device", "virtio-net,netdev=user.0", ) } diff --git a/drivers/shared/resolvconf/mount.go b/drivers/shared/resolvconf/mount.go new file mode 100644 index 000000000000..02393478efcd --- /dev/null +++ b/drivers/shared/resolvconf/mount.go @@ -0,0 +1,83 @@ +package resolvconf + +import ( + "io" + "os" + "path/filepath" + + dresolvconf "github.com/docker/libnetwork/resolvconf" + "github.com/docker/libnetwork/types" + "github.com/hashicorp/nomad/plugins/drivers" +) + +func GenerateDNSMount(taskDir string, conf *drivers.DNSConfig) (*drivers.MountConfig, error) { + var nSearches, nServers, nOptions int + path := filepath.Join(taskDir, "resolv.conf") + mount := &drivers.MountConfig{ + TaskPath: "/etc/resolv.conf", + HostPath: path, + Readonly: true, + PropagationMode: "private", + } + if conf != nil { + nServers = len(conf.Servers) + nSearches = len(conf.Searches) + nOptions = len(conf.Options) + } + + // Use system dns if no configuration is given + if nServers == 0 && nSearches == 0 && nOptions == 0 { + if err := copySystemDNS(path); err != nil { + return nil, err + } + + return mount, nil + } + + currRC, err := dresolvconf.Get() + if err != nil { + return nil, err + } + + var ( + dnsList = dresolvconf.GetNameservers(currRC.Content, types.IP) + dnsSearchList = dresolvconf.GetSearchDomains(currRC.Content) + dnsOptionsList = dresolvconf.GetOptions(currRC.Content) + ) + if nServers > 0 { + dnsList = conf.Servers + } + if nSearches > 0 { + dnsSearchList = conf.Searches + } + if nOptions > 0 { + dnsOptionsList = conf.Options + } + + _, err = dresolvconf.Build(path, dnsList, dnsSearchList, dnsOptionsList) + if err != nil { + return nil, err + } + + return mount, nil +} + +func copySystemDNS(dest string) error { + in, err := os.Open(dresolvconf.Path()) + if err != nil { + return err + } + defer in.Close() + + out, err := os.Create(dest) + if err != nil { + return err + } + defer func() { + out.Sync() + out.Close() + }() + + _, err = io.Copy(out, in) + return err +} diff --git a/drivers/shared/resolvconf/mount_unix_test.go b/drivers/shared/resolvconf/mount_unix_test.go new file mode 100644 index 000000000000..76154f5909ba --- /dev/null +++ b/drivers/shared/resolvconf/mount_unix_test.go @@ -0,0 +1,30 @@ +// +build !windows + +package resolvconf + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func Test_copySystemDNS(t *testing.T) { + require := require.New(t) + data, err := ioutil.ReadFile("/etc/resolv.conf") + require.NoError(err) + + tmp, err := ioutil.TempDir("", "copySystemDNS_Test") + require.NoError(err) + defer os.RemoveAll(tmp) + dest := filepath.Join(tmp, "resolv.conf") + + require.NoError(copySystemDNS(dest)) + require.FileExists(dest) + + tmpResolv, err := ioutil.ReadFile(dest) + require.NoError(err) + require.Equal(data, tmpResolv) +} diff --git a/go.mod b/go.mod index 6a9e11d8354e..3f41c1a4d46a 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/container-storage-interface/spec v1.2.0-rc1.0.20191021210849-a33ece0a8a9f github.com/containerd/console v1.0.0 // indirect github.com/containerd/go-cni v0.0.0-20190904155053-d20b7eebc7ee - github.com/containernetworking/cni v0.7.2-0.20190612152420-dc953e2fd91f // indirect + github.com/containernetworking/cni v0.7.2-0.20190612152420-dc953e2fd91f github.com/containernetworking/plugins v0.7.3-0.20190501191748-2d6d46d308b2 github.com/coreos/go-iptables v0.4.3-0.20190724151750-969b135e941d github.com/coreos/go-semver v0.3.0 @@ -41,6 +41,7 @@ require ( github.com/docker/docker v17.12.0-ce-rc1.0.20200330121334-7f8b4b621b5d+incompatible github.com/docker/docker-credential-helpers v0.6.2-0.20180719074751-73e5f5dbfea3 // indirect github.com/docker/go-units v0.4.0 + github.com/docker/libnetwork v0.8.0-dev.2.0.20200612180813-9e99af28df21 github.com/dustin/go-humanize v1.0.0 github.com/elazarl/go-bindata-assetfs v1.0.1-0.20200509193318-234c15e7648f github.com/fatih/color v1.9.0 @@ -87,6 +88,7 @@ require ( github.com/hashicorp/vault/sdk v0.1.14-0.20190730042320-0dc007d98cc8 github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d github.com/hpcloud/tail v1.0.1-0.20170814160653-37f427138745 + github.com/ishidawataru/sctp v0.0.0-20191218070446-00ab2ac2db07 // indirect github.com/joyent/triton-go v0.0.0-20190112182421-51ffac552869 // indirect github.com/kr/pretty v0.2.0 github.com/kr/pty v1.1.5 diff --git a/go.sum b/go.sum index 344ae7016a1f..d86cb28afedb 100644 --- a/go.sum +++ b/go.sum @@ -119,6 +119,7 @@ github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/container-storage-interface/spec v1.2.0-rc1.0.20191021210849-a33ece0a8a9f h1:m2LYF3fo9IPapVt5FGRVw5bJPmlWqWIezB0jkQh03Zo= github.com/container-storage-interface/spec v1.2.0-rc1.0.20191021210849-a33ece0a8a9f/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= +github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f h1:tSNMc+rJDfmYntojat8lljbt1mgKNpTxUZJsSzJ9Y1s= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v1.0.0 h1:fU3UuQapBs+zLJu82NhR11Rif1ny2zfMMAyPJzSN5tQ= @@ -183,12 +184,15 @@ github.com/docker/docker-credential-helpers v0.6.2-0.20180719074751-73e5f5dbfea3 github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916 h1:yWHOI+vFjEsAakUTSrtqc/SAHrhSkmn48pqjidZX3QA= github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/libnetwork v0.8.0-dev.2.0.20200612180813-9e99af28df21 h1:RvyaXv8RgMpJIT73oQFOj4Vos1c1rwJcIm3sPHdd1Js= +github.com/docker/libnetwork v0.8.0-dev.2.0.20200612180813-9e99af28df21/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -435,6 +439,8 @@ github.com/hpcloud/tail v1.0.1-0.20170814160653-37f427138745/go.mod h1:ab1qPbhIp github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/ishidawataru/sctp v0.0.0-20191218070446-00ab2ac2db07 h1:rw3IAne6CDuVFlZbPOkA7bhxlqawFh7RJJ+CejfMaxE= +github.com/ishidawataru/sctp v0.0.0-20191218070446-00ab2ac2db07/go.mod h1:co9pwDoBCm1kGxawmb4sPq0cSIOOWNPT4KnHotMP1Zg= github.com/jarcoal/httpmock v0.0.0-20180424175123-9c70cfe4a1da h1:FjHUJJ7oBW4G/9j1KzlHaXL09LyMVM9rupS39lncbXk= github.com/jarcoal/httpmock v0.0.0-20180424175123-9c70cfe4a1da/go.mod h1:ks+b9deReOc7jgqp+e7LuFiCBH6Rm5hL32cLcEAArb4= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= @@ -447,6 +453,7 @@ github.com/joyent/triton-go v0.0.0-20190112182421-51ffac552869 h1:BvV6PYcRz0yGnW github.com/joyent/triton-go v0.0.0-20190112182421-51ffac552869/go.mod h1:U+RSyWxWd04xTqnuOQxnai7XGS2PrPY2cfGoDKtMHjA= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024 h1:rBMNdlhTLzJjJSDIjNEXX1Pz3Hmwmz91v+zycvx9PJc= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -532,8 +539,10 @@ github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE= github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= diff --git a/jobspec/parse_network.go b/jobspec/parse_network.go index 6cd2ca6cbaf9..e72719309f2f 100644 --- a/jobspec/parse_network.go +++ b/jobspec/parse_network.go @@ -22,6 +22,7 @@ func ParseNetwork(o *ast.ObjectList) (*api.NetworkResource, error) { valid := []string{ "mode", "mbits", + "dns", "port", } if err := helper.CheckHCLKeys(o.Items[0].Val, valid); err != nil { @@ -33,6 +34,8 @@ func ParseNetwork(o *ast.ObjectList) (*api.NetworkResource, error) { if err := hcl.DecodeObject(&m, o.Items[0].Val); err != nil { return nil, err } + + delete(m, "dns") if err := mapstructure.WeakDecode(m, &r); err != nil { return nil, err } @@ -47,26 +50,40 @@ func ParseNetwork(o *ast.ObjectList) (*api.NetworkResource, error) { return nil, multierror.Prefix(err, "network, ports ->") } + // Filter dns + if dns := networkObj.Filter("dns"); len(dns.Items) > 0 { + if len(dns.Items) > 1 { + return nil, multierror.Prefix(fmt.Errorf("cannot have more than 1 dns stanza"), "network ->") + } + + d, err := parseDNS(dns.Items[0]) + if err != nil { + return nil, multierror.Prefix(err, "network ->") + } + + r.DNS = d + } + return &r, nil } func parsePorts(networkObj *ast.ObjectList, nw *api.NetworkResource) error { - // Check for invalid keys - valid := []string{ - "mbits", - "port", - "mode", - } - if err := helper.CheckHCLKeys(networkObj, valid); err != nil { - return err - } - portsObjList := networkObj.Filter("port") knownPortLabels := make(map[string]bool) for _, port := range portsObjList.Items { if len(port.Keys) == 0 { return fmt.Errorf("ports must be named") } + + // check for invalid keys + valid := []string{ + "static", + "to", + } + if err := helper.CheckHCLKeys(port.Val, valid); err != nil { + return err + } + label := port.Keys[0].Token.Value().(string) if !reDynamicPorts.MatchString(label) { return errPortLabel @@ -93,3 +110,27 @@ func parsePorts(networkObj *ast.ObjectList, nw *api.NetworkResource) error { } return nil } + +func parseDNS(dns *ast.ObjectItem) (*api.DNSConfig, error) { + valid := []string{ + "servers", + "searches", + "options", + } + + if err := helper.CheckHCLKeys(dns.Val, valid); err != nil { + return nil, multierror.Prefix(err, "dns ->") + } + + var dnsCfg api.DNSConfig + var m map[string]interface{} + if err := hcl.DecodeObject(&m, dns.Val); err != nil { + return nil, err + } + + if err := mapstructure.WeakDecode(m, &dnsCfg); err != nil { + return nil, err + } + + return &dnsCfg, nil +} diff --git a/jobspec/parse_test.go b/jobspec/parse_test.go index 426ccbd34566..ed36283a091d 100644 --- a/jobspec/parse_test.go +++ b/jobspec/parse_test.go @@ -1018,6 +1018,10 @@ func TestParse(t *testing.T) { To: 8080, }, }, + DNS: &api.DNSConfig{ + Servers: []string{"8.8.8.8"}, + Options: []string{"ndots:2", "edns0"}, + }, }, }, Services: []*api.Service{ diff --git a/jobspec/test-fixtures/tg-network.hcl b/jobspec/test-fixtures/tg-network.hcl index 538f49f3acfe..faaa79f94e5e 100644 --- a/jobspec/test-fixtures/tg-network.hcl +++ b/jobspec/test-fixtures/tg-network.hcl @@ -12,6 +12,11 @@ job "foo" { static = 80 to = 8080 } + + dns { + servers = ["8.8.8.8"] + options = ["ndots:2", "edns0"] + } } service { diff --git a/nomad/mock/mock.go b/nomad/mock/mock.go index c63530a49c9a..b070a2ffa13c 100644 --- a/nomad/mock/mock.go +++ b/nomad/mock/mock.go @@ -65,6 +65,7 @@ func Node() *structs.Node { }, Networks: []*structs.NetworkResource{ { + Mode: "host", Device: "eth0", CIDR: "192.168.0.100/32", MBits: 1000, diff --git a/nomad/structs/diff.go b/nomad/structs/diff.go index 2ed549a94aeb..93e86eb7e81d 100644 --- a/nomad/structs/diff.go +++ b/nomad/structs/diff.go @@ -1206,6 +1206,50 @@ func (r *NetworkResource) Diff(other *NetworkResource, contextual bool) *ObjectD diff.Objects = append(diff.Objects, dynPorts...) } + if dnsDiff := r.DNS.Diff(other.DNS, contextual); dnsDiff != nil { + diff.Objects = append(diff.Objects, dnsDiff) + } + + return diff +} + +// Diff returns a diff of two DNSConfig structs +func (c *DNSConfig) Diff(other *DNSConfig, contextual bool) *ObjectDiff { + if reflect.DeepEqual(c, other) { + return nil + } + + flatten := func(conf *DNSConfig) map[string]string { + m := map[string]string{} + if len(conf.Servers) > 0 { + m["Servers"] = strings.Join(conf.Servers, ",") + } + if len(conf.Searches) > 0 { + m["Searches"] = strings.Join(conf.Searches, ",") + } + if len(conf.Options) > 0 { + m["Options"] = strings.Join(conf.Options, ",") + } + return m + } + + diff := &ObjectDiff{Type: DiffTypeNone, Name: "DNS"} + var oldPrimitiveFlat, newPrimitiveFlat map[string]string + if c == nil { + diff.Type = DiffTypeAdded + newPrimitiveFlat = flatten(other) + } else if other == nil { + diff.Type = DiffTypeDeleted + oldPrimitiveFlat = flatten(c) + } else { + diff.Type = DiffTypeEdited + oldPrimitiveFlat = flatten(c) + newPrimitiveFlat = flatten(other) + } + + // Diff the primitive fields. + diff.Fields = fieldDiffs(oldPrimitiveFlat, newPrimitiveFlat, contextual) + return diff } diff --git a/nomad/structs/diff_test.go b/nomad/structs/diff_test.go index 510cc92afef0..84d6a27dd596 100644 --- a/nomad/structs/diff_test.go +++ b/nomad/structs/diff_test.go @@ -2935,6 +2935,9 @@ func TestTaskGroupDiff(t *testing.T) { To: 8081, }, }, + DNS: &DNSConfig{ + Servers: []string{"1.1.1.1"}, + }, }, }, }, @@ -2977,6 +2980,18 @@ func TestTaskGroupDiff(t *testing.T) { }, }, }, + { + Type: DiffTypeAdded, + Name: "DNS", + Fields: []*FieldDiff{ + { + Type: DiffTypeAdded, + Name: "Servers", + Old: "", + New: "1.1.1.1", + }, + }, + }, }, }, { diff --git a/nomad/structs/network.go b/nomad/structs/network.go index 19aa47197cc3..f89508411f08 100644 --- a/nomad/structs/network.go +++ b/nomad/structs/network.go @@ -290,6 +290,7 @@ func (idx *NetworkIndex) AssignNetwork(ask *NetworkResource) (out *NetworkResour Device: n.Device, IP: ipStr, MBits: ask.MBits, + DNS: ask.DNS, ReservedPorts: ask.ReservedPorts, DynamicPorts: ask.DynamicPorts, } diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 875ebb826548..f11a46a0f048 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -12,6 +12,7 @@ import ( "encoding/hex" "errors" "fmt" + "hash/crc32" "math" "net" "os" @@ -2246,65 +2247,42 @@ type Port struct { To int } +type DNSConfig struct { + Servers []string + Searches []string + Options []string +} + // NetworkResource is used to represent available network // resources type NetworkResource struct { - Mode string // Mode of the network - Device string // Name of the device - CIDR string // CIDR block of addresses - IP string // Host IP address - MBits int // Throughput - ReservedPorts []Port // Host Reserved ports - DynamicPorts []Port // Host Dynamically assigned ports + Mode string // Mode of the network + Device string // Name of the device + CIDR string // CIDR block of addresses + IP string // Host IP address + MBits int // Throughput + DNS *DNSConfig // DNS Configuration + ReservedPorts []Port // Host Reserved ports + DynamicPorts []Port // Host Dynamically assigned ports } -func (nr *NetworkResource) Equals(other *NetworkResource) bool { - if nr.Mode != other.Mode { - return false - } - - if nr.Device != other.Device { - return false - } - - if nr.CIDR != other.CIDR { - return false - } - - if nr.IP != other.IP { - return false - } - - if nr.MBits != other.MBits { - return false - } - - if len(nr.ReservedPorts) != len(other.ReservedPorts) { - return false - } +func (nr *NetworkResource) Hash() uint32 { + var data []byte + data = append(data, []byte(fmt.Sprintf("%s%s%s%s%d", nr.Mode, nr.Device, nr.CIDR, nr.IP, nr.MBits))...) for i, port := range nr.ReservedPorts { - if len(other.ReservedPorts) <= i { - return false - } - if port != other.ReservedPorts[i] { - return false - } + data = append(data, []byte(fmt.Sprintf("r%d%s%d%d", i, port.Label, port.Value, port.To))...) } - if len(nr.DynamicPorts) != len(other.DynamicPorts) { - return false - } for i, port := range nr.DynamicPorts { - if len(other.DynamicPorts) <= i { - return false - } - if port != other.DynamicPorts[i] { - return false - } + data = append(data, []byte(fmt.Sprintf("d%d%s%d%d", i, port.Label, port.Value, port.To))...) } - return true + return crc32.ChecksumIEEE(data) +} + +func (nr *NetworkResource) Equals(other *NetworkResource) bool { + return nr.Hash() == other.Hash() } func (n *NetworkResource) Canonicalize() { @@ -2606,7 +2584,7 @@ func (n *NodeResources) Merge(o *NodeResources) { n.Disk.Merge(&o.Disk) if len(o.Networks) != 0 { - n.Networks = o.Networks + n.Networks = append(n.Networks, o.Networks...) } if len(o.Devices) != 0 { diff --git a/nomad/structs/structs_test.go b/nomad/structs/structs_test.go index 6a8664d75128..1f57cb4a3545 100644 --- a/nomad/structs/structs_test.go +++ b/nomad/structs/structs_test.go @@ -5281,6 +5281,50 @@ func TestMultiregion_CopyCanonicalize(t *testing.T) { require.False(old.Diff(nonEmptyOld)) } +func TestNodeResources_Merge(t *testing.T) { + res := &NodeResources{ + Cpu: NodeCpuResources{ + CpuShares: int64(32000), + }, + Memory: NodeMemoryResources{ + MemoryMB: int64(64000), + }, + Networks: Networks{ + { + Device: "foo", + }, + }, + } + + res.Merge(&NodeResources{ + Memory: NodeMemoryResources{ + MemoryMB: int64(100000), + }, + Networks: Networks{ + { + Mode: "foo/bar", + }, + }, + }) + + require.Exactly(t, &NodeResources{ + Cpu: NodeCpuResources{ + CpuShares: int64(32000), + }, + Memory: NodeMemoryResources{ + MemoryMB: int64(100000), + }, + Networks: Networks{ + { + Device: "foo", + }, + { + Mode: "foo/bar", + }, + }, + }, res) +} + func TestMultiregion_Validate(t *testing.T) { require := require.New(t) cases := []struct { diff --git a/plugins/drivers/driver.go b/plugins/drivers/driver.go index 89e3061720ea..5cf4cf7fe8ca 100644 --- a/plugins/drivers/driver.go +++ b/plugins/drivers/driver.go @@ -206,6 +206,34 @@ type TerminalSize struct { Width int } +type DNSConfig struct { + Servers []string + Searches []string + Options []string +} + +func (c *DNSConfig) Copy() *DNSConfig { + if c == nil { + return nil + } + + cfg := new(DNSConfig) + if len(c.Servers) > 0 { + cfg.Servers = make([]string, len(c.Servers)) + copy(cfg.Servers, c.Servers) + } + if len(c.Searches) > 0 { + cfg.Searches = make([]string, len(c.Searches)) + copy(cfg.Searches, c.Searches) + } + if len(c.Options) > 0 { + cfg.Options = make([]string, len(c.Options)) + copy(cfg.Options, c.Options) + } + + return cfg +} + type TaskConfig struct { ID string JobName string @@ -223,6 +251,7 @@ type TaskConfig struct { StderrPath string AllocID string NetworkIsolation *NetworkIsolationSpec + DNS *DNSConfig } func (tc *TaskConfig) Copy() *TaskConfig { @@ -234,6 +263,7 @@ func (tc *TaskConfig) Copy() *TaskConfig { c.Env = helper.CopyMapStringString(c.Env) c.DeviceEnv = helper.CopyMapStringString(c.DeviceEnv) c.Resources = tc.Resources.Copy() + c.DNS = tc.DNS.Copy() if c.Devices != nil { dc := make([]*DeviceConfig, len(c.Devices)) diff --git a/plugins/drivers/proto/driver.pb.go b/plugins/drivers/proto/driver.pb.go index 823b5110fb2d..40a12e0dbbce 100644 --- a/plugins/drivers/proto/driver.pb.go +++ b/plugins/drivers/proto/driver.pb.go @@ -233,7 +233,7 @@ func (x CPUUsage_Fields) String() string { } func (CPUUsage_Fields) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_4a8f45747846a74d, []int{51, 0} + return fileDescriptor_4a8f45747846a74d, []int{52, 0} } type MemoryUsage_Fields int32 @@ -273,7 +273,7 @@ func (x MemoryUsage_Fields) String() string { } func (MemoryUsage_Fields) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_4a8f45747846a74d, []int{52, 0} + return fileDescriptor_4a8f45747846a74d, []int{53, 0} } type TaskConfigSchemaRequest struct { @@ -1964,6 +1964,61 @@ func (m *NetworkIsolationSpec) GetLabels() map[string]string { return nil } +type DNSConfig struct { + Servers []string `protobuf:"bytes,1,rep,name=servers,proto3" json:"servers,omitempty"` + Searches []string `protobuf:"bytes,2,rep,name=searches,proto3" json:"searches,omitempty"` + Options []string `protobuf:"bytes,3,rep,name=options,proto3" json:"options,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DNSConfig) Reset() { *m = DNSConfig{} } +func (m *DNSConfig) String() string { return proto.CompactTextString(m) } +func (*DNSConfig) ProtoMessage() {} +func (*DNSConfig) Descriptor() ([]byte, []int) { + return fileDescriptor_4a8f45747846a74d, []int{34} +} + +func (m *DNSConfig) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DNSConfig.Unmarshal(m, b) +} +func (m *DNSConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DNSConfig.Marshal(b, m, deterministic) +} +func (m *DNSConfig) XXX_Merge(src proto.Message) { + xxx_messageInfo_DNSConfig.Merge(m, src) +} +func (m *DNSConfig) XXX_Size() int { + return xxx_messageInfo_DNSConfig.Size(m) +} +func (m *DNSConfig) XXX_DiscardUnknown() { + xxx_messageInfo_DNSConfig.DiscardUnknown(m) +} + +var xxx_messageInfo_DNSConfig proto.InternalMessageInfo + +func (m *DNSConfig) GetServers() []string { + if m != nil { + return m.Servers + } + return nil +} + +func (m *DNSConfig) GetSearches() []string { + if m != nil { + return m.Searches + } + return nil +} + +func (m *DNSConfig) GetOptions() []string { + if m != nil { + return m.Options + } + return nil +} + type TaskConfig struct { // Id of the task, recommended to the globally unique, must be unique to the driver. Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` @@ -2003,16 +2058,18 @@ type TaskConfig struct { // NetworkIsolationSpec specifies the configuration for the network namespace // to use for the task. *Only supported on Linux NetworkIsolationSpec *NetworkIsolationSpec `protobuf:"bytes,16,opt,name=network_isolation_spec,json=networkIsolationSpec,proto3" json:"network_isolation_spec,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + // DNSConfig is the configuration for task DNS resolvers and other options + Dns *DNSConfig `protobuf:"bytes,17,opt,name=dns,proto3" json:"dns,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *TaskConfig) Reset() { *m = TaskConfig{} } func (m *TaskConfig) String() string { return proto.CompactTextString(m) } func (*TaskConfig) ProtoMessage() {} func (*TaskConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_4a8f45747846a74d, []int{34} + return fileDescriptor_4a8f45747846a74d, []int{35} } func (m *TaskConfig) XXX_Unmarshal(b []byte) error { @@ -2145,6 +2202,13 @@ func (m *TaskConfig) GetNetworkIsolationSpec() *NetworkIsolationSpec { return nil } +func (m *TaskConfig) GetDns() *DNSConfig { + if m != nil { + return m.Dns + } + return nil +} + type Resources struct { // AllocatedResources are the resources set for the task AllocatedResources *AllocatedTaskResources `protobuf:"bytes,1,opt,name=allocated_resources,json=allocatedResources,proto3" json:"allocated_resources,omitempty"` @@ -2159,7 +2223,7 @@ func (m *Resources) Reset() { *m = Resources{} } func (m *Resources) String() string { return proto.CompactTextString(m) } func (*Resources) ProtoMessage() {} func (*Resources) Descriptor() ([]byte, []int) { - return fileDescriptor_4a8f45747846a74d, []int{35} + return fileDescriptor_4a8f45747846a74d, []int{36} } func (m *Resources) XXX_Unmarshal(b []byte) error { @@ -2207,7 +2271,7 @@ func (m *AllocatedTaskResources) Reset() { *m = AllocatedTaskResources{} func (m *AllocatedTaskResources) String() string { return proto.CompactTextString(m) } func (*AllocatedTaskResources) ProtoMessage() {} func (*AllocatedTaskResources) Descriptor() ([]byte, []int) { - return fileDescriptor_4a8f45747846a74d, []int{36} + return fileDescriptor_4a8f45747846a74d, []int{37} } func (m *AllocatedTaskResources) XXX_Unmarshal(b []byte) error { @@ -2260,7 +2324,7 @@ func (m *AllocatedCpuResources) Reset() { *m = AllocatedCpuResources{} } func (m *AllocatedCpuResources) String() string { return proto.CompactTextString(m) } func (*AllocatedCpuResources) ProtoMessage() {} func (*AllocatedCpuResources) Descriptor() ([]byte, []int) { - return fileDescriptor_4a8f45747846a74d, []int{37} + return fileDescriptor_4a8f45747846a74d, []int{38} } func (m *AllocatedCpuResources) XXX_Unmarshal(b []byte) error { @@ -2299,7 +2363,7 @@ func (m *AllocatedMemoryResources) Reset() { *m = AllocatedMemoryResourc func (m *AllocatedMemoryResources) String() string { return proto.CompactTextString(m) } func (*AllocatedMemoryResources) ProtoMessage() {} func (*AllocatedMemoryResources) Descriptor() ([]byte, []int) { - return fileDescriptor_4a8f45747846a74d, []int{38} + return fileDescriptor_4a8f45747846a74d, []int{39} } func (m *AllocatedMemoryResources) XXX_Unmarshal(b []byte) error { @@ -2343,7 +2407,7 @@ func (m *NetworkResource) Reset() { *m = NetworkResource{} } func (m *NetworkResource) String() string { return proto.CompactTextString(m) } func (*NetworkResource) ProtoMessage() {} func (*NetworkResource) Descriptor() ([]byte, []int) { - return fileDescriptor_4a8f45747846a74d, []int{39} + return fileDescriptor_4a8f45747846a74d, []int{40} } func (m *NetworkResource) XXX_Unmarshal(b []byte) error { @@ -2418,7 +2482,7 @@ func (m *NetworkPort) Reset() { *m = NetworkPort{} } func (m *NetworkPort) String() string { return proto.CompactTextString(m) } func (*NetworkPort) ProtoMessage() {} func (*NetworkPort) Descriptor() ([]byte, []int) { - return fileDescriptor_4a8f45747846a74d, []int{40} + return fileDescriptor_4a8f45747846a74d, []int{41} } func (m *NetworkPort) XXX_Unmarshal(b []byte) error { @@ -2479,7 +2543,7 @@ func (m *LinuxResources) Reset() { *m = LinuxResources{} } func (m *LinuxResources) String() string { return proto.CompactTextString(m) } func (*LinuxResources) ProtoMessage() {} func (*LinuxResources) Descriptor() ([]byte, []int) { - return fileDescriptor_4a8f45747846a74d, []int{41} + return fileDescriptor_4a8f45747846a74d, []int{42} } func (m *LinuxResources) XXX_Unmarshal(b []byte) error { @@ -2572,7 +2636,7 @@ func (m *Mount) Reset() { *m = Mount{} } func (m *Mount) String() string { return proto.CompactTextString(m) } func (*Mount) ProtoMessage() {} func (*Mount) Descriptor() ([]byte, []int) { - return fileDescriptor_4a8f45747846a74d, []int{42} + return fileDescriptor_4a8f45747846a74d, []int{43} } func (m *Mount) XXX_Unmarshal(b []byte) error { @@ -2636,7 +2700,7 @@ func (m *Device) Reset() { *m = Device{} } func (m *Device) String() string { return proto.CompactTextString(m) } func (*Device) ProtoMessage() {} func (*Device) Descriptor() ([]byte, []int) { - return fileDescriptor_4a8f45747846a74d, []int{43} + return fileDescriptor_4a8f45747846a74d, []int{44} } func (m *Device) XXX_Unmarshal(b []byte) error { @@ -2698,7 +2762,7 @@ func (m *TaskHandle) Reset() { *m = TaskHandle{} } func (m *TaskHandle) String() string { return proto.CompactTextString(m) } func (*TaskHandle) ProtoMessage() {} func (*TaskHandle) Descriptor() ([]byte, []int) { - return fileDescriptor_4a8f45747846a74d, []int{44} + return fileDescriptor_4a8f45747846a74d, []int{45} } func (m *TaskHandle) XXX_Unmarshal(b []byte) error { @@ -2766,7 +2830,7 @@ func (m *NetworkOverride) Reset() { *m = NetworkOverride{} } func (m *NetworkOverride) String() string { return proto.CompactTextString(m) } func (*NetworkOverride) ProtoMessage() {} func (*NetworkOverride) Descriptor() ([]byte, []int) { - return fileDescriptor_4a8f45747846a74d, []int{45} + return fileDescriptor_4a8f45747846a74d, []int{46} } func (m *NetworkOverride) XXX_Unmarshal(b []byte) error { @@ -2825,7 +2889,7 @@ func (m *ExitResult) Reset() { *m = ExitResult{} } func (m *ExitResult) String() string { return proto.CompactTextString(m) } func (*ExitResult) ProtoMessage() {} func (*ExitResult) Descriptor() ([]byte, []int) { - return fileDescriptor_4a8f45747846a74d, []int{46} + return fileDescriptor_4a8f45747846a74d, []int{47} } func (m *ExitResult) XXX_Unmarshal(b []byte) error { @@ -2889,7 +2953,7 @@ func (m *TaskStatus) Reset() { *m = TaskStatus{} } func (m *TaskStatus) String() string { return proto.CompactTextString(m) } func (*TaskStatus) ProtoMessage() {} func (*TaskStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_4a8f45747846a74d, []int{47} + return fileDescriptor_4a8f45747846a74d, []int{48} } func (m *TaskStatus) XXX_Unmarshal(b []byte) error { @@ -2965,7 +3029,7 @@ func (m *TaskDriverStatus) Reset() { *m = TaskDriverStatus{} } func (m *TaskDriverStatus) String() string { return proto.CompactTextString(m) } func (*TaskDriverStatus) ProtoMessage() {} func (*TaskDriverStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_4a8f45747846a74d, []int{48} + return fileDescriptor_4a8f45747846a74d, []int{49} } func (m *TaskDriverStatus) XXX_Unmarshal(b []byte) error { @@ -3011,7 +3075,7 @@ func (m *TaskStats) Reset() { *m = TaskStats{} } func (m *TaskStats) String() string { return proto.CompactTextString(m) } func (*TaskStats) ProtoMessage() {} func (*TaskStats) Descriptor() ([]byte, []int) { - return fileDescriptor_4a8f45747846a74d, []int{49} + return fileDescriptor_4a8f45747846a74d, []int{50} } func (m *TaskStats) XXX_Unmarshal(b []byte) error { @@ -3074,7 +3138,7 @@ func (m *TaskResourceUsage) Reset() { *m = TaskResourceUsage{} } func (m *TaskResourceUsage) String() string { return proto.CompactTextString(m) } func (*TaskResourceUsage) ProtoMessage() {} func (*TaskResourceUsage) Descriptor() ([]byte, []int) { - return fileDescriptor_4a8f45747846a74d, []int{50} + return fileDescriptor_4a8f45747846a74d, []int{51} } func (m *TaskResourceUsage) XXX_Unmarshal(b []byte) error { @@ -3127,7 +3191,7 @@ func (m *CPUUsage) Reset() { *m = CPUUsage{} } func (m *CPUUsage) String() string { return proto.CompactTextString(m) } func (*CPUUsage) ProtoMessage() {} func (*CPUUsage) Descriptor() ([]byte, []int) { - return fileDescriptor_4a8f45747846a74d, []int{51} + return fileDescriptor_4a8f45747846a74d, []int{52} } func (m *CPUUsage) XXX_Unmarshal(b []byte) error { @@ -3216,7 +3280,7 @@ func (m *MemoryUsage) Reset() { *m = MemoryUsage{} } func (m *MemoryUsage) String() string { return proto.CompactTextString(m) } func (*MemoryUsage) ProtoMessage() {} func (*MemoryUsage) Descriptor() ([]byte, []int) { - return fileDescriptor_4a8f45747846a74d, []int{52} + return fileDescriptor_4a8f45747846a74d, []int{53} } func (m *MemoryUsage) XXX_Unmarshal(b []byte) error { @@ -3315,7 +3379,7 @@ func (m *DriverTaskEvent) Reset() { *m = DriverTaskEvent{} } func (m *DriverTaskEvent) String() string { return proto.CompactTextString(m) } func (*DriverTaskEvent) ProtoMessage() {} func (*DriverTaskEvent) Descriptor() ([]byte, []int) { - return fileDescriptor_4a8f45747846a74d, []int{53} + return fileDescriptor_4a8f45747846a74d, []int{54} } func (m *DriverTaskEvent) XXX_Unmarshal(b []byte) error { @@ -3425,6 +3489,7 @@ func init() { proto.RegisterType((*DriverCapabilities)(nil), "hashicorp.nomad.plugins.drivers.proto.DriverCapabilities") proto.RegisterType((*NetworkIsolationSpec)(nil), "hashicorp.nomad.plugins.drivers.proto.NetworkIsolationSpec") proto.RegisterMapType((map[string]string)(nil), "hashicorp.nomad.plugins.drivers.proto.NetworkIsolationSpec.LabelsEntry") + proto.RegisterType((*DNSConfig)(nil), "hashicorp.nomad.plugins.drivers.proto.DNSConfig") proto.RegisterType((*TaskConfig)(nil), "hashicorp.nomad.plugins.drivers.proto.TaskConfig") proto.RegisterMapType((map[string]string)(nil), "hashicorp.nomad.plugins.drivers.proto.TaskConfig.DeviceEnvEntry") proto.RegisterMapType((map[string]string)(nil), "hashicorp.nomad.plugins.drivers.proto.TaskConfig.EnvEntry") @@ -3458,232 +3523,235 @@ func init() { } var fileDescriptor_4a8f45747846a74d = []byte{ - // 3586 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x5a, 0x4f, 0x73, 0xdb, 0x48, - 0x76, 0x17, 0xf8, 0x4f, 0xe4, 0x23, 0x45, 0x41, 0x2d, 0xc9, 0x43, 0x73, 0x92, 0x8c, 0x17, 0x55, - 0x9b, 0x52, 0xed, 0x8e, 0xa9, 0x19, 0x6d, 0xc5, 0xb6, 0xbc, 0xf6, 0xda, 0x1c, 0x8a, 0x96, 0x34, - 0x96, 0x48, 0xa5, 0x49, 0x95, 0xd7, 0x71, 0x76, 0x10, 0x08, 0x68, 0x93, 0xb0, 0x88, 0x3f, 0x06, - 0x40, 0x59, 0xda, 0x54, 0x2a, 0xa9, 0x4d, 0x55, 0x6a, 0x53, 0x95, 0x54, 0x72, 0x99, 0xec, 0x25, - 0xa7, 0xcd, 0x31, 0xf9, 0x00, 0xa9, 0xa4, 0xf6, 0x9c, 0x2f, 0x90, 0x5b, 0x72, 0xc9, 0x2d, 0x97, - 0x1c, 0xf2, 0x0d, 0xb6, 0xfa, 0x0f, 0x40, 0x40, 0xa4, 0xc7, 0x20, 0xe5, 0x13, 0xd0, 0xaf, 0xbb, - 0x7f, 0xfd, 0xf0, 0xde, 0xeb, 0x7e, 0xaf, 0x1f, 0x1e, 0x28, 0xee, 0x68, 0x3c, 0x30, 0x6d, 0x7f, - 0xdb, 0xf0, 0xcc, 0x0b, 0xe2, 0xf9, 0xdb, 0xae, 0xe7, 0x04, 0x8e, 0x68, 0x35, 0x58, 0x03, 0x7d, - 0x7f, 0xa8, 0xf9, 0x43, 0x53, 0x77, 0x3c, 0xb7, 0x61, 0x3b, 0x96, 0x66, 0x34, 0xc4, 0x9c, 0x86, - 0x98, 0xc3, 0x87, 0xd5, 0x7f, 0x6f, 0xe0, 0x38, 0x83, 0x11, 0xe1, 0x08, 0x67, 0xe3, 0xd7, 0xdb, - 0xc6, 0xd8, 0xd3, 0x02, 0xd3, 0xb1, 0x45, 0xff, 0x67, 0xd7, 0xfb, 0x03, 0xd3, 0x22, 0x7e, 0xa0, - 0x59, 0xae, 0x18, 0xf0, 0x74, 0x60, 0x06, 0xc3, 0xf1, 0x59, 0x43, 0x77, 0xac, 0xed, 0x68, 0xc9, - 0x6d, 0xb6, 0xe4, 0x76, 0xc8, 0xa6, 0x3f, 0xd4, 0x3c, 0x62, 0x6c, 0x0f, 0xf5, 0x91, 0xef, 0x12, - 0x9d, 0x3e, 0x55, 0xfa, 0x22, 0x10, 0xf6, 0xd3, 0x23, 0xf8, 0x81, 0x37, 0xd6, 0x83, 0xf0, 0x7b, - 0xb5, 0x20, 0xf0, 0xcc, 0xb3, 0x71, 0x40, 0x38, 0x90, 0x72, 0x1b, 0x3e, 0xe9, 0x6b, 0xfe, 0x79, - 0xcb, 0xb1, 0x5f, 0x9b, 0x83, 0x9e, 0x3e, 0x24, 0x96, 0x86, 0xc9, 0xdb, 0x31, 0xf1, 0x03, 0xe5, - 0x8f, 0xa1, 0x36, 0xdd, 0xe5, 0xbb, 0x8e, 0xed, 0x13, 0xf4, 0x14, 0x72, 0x94, 0x9b, 0x9a, 0x74, - 0x47, 0xda, 0x2a, 0xef, 0x7c, 0xde, 0x78, 0x9f, 0xe0, 0x38, 0x0f, 0x0d, 0xf1, 0x15, 0x8d, 0x9e, - 0x4b, 0x74, 0xcc, 0x66, 0x2a, 0x9b, 0xb0, 0xde, 0xd2, 0x5c, 0xed, 0xcc, 0x1c, 0x99, 0x81, 0x49, - 0xfc, 0x70, 0xd1, 0x31, 0x6c, 0x24, 0xc9, 0x62, 0xc1, 0x9f, 0x41, 0x45, 0x8f, 0xd1, 0xc5, 0xc2, - 0xbb, 0x8d, 0x54, 0x1a, 0x6b, 0xec, 0xb1, 0x56, 0x02, 0x38, 0x01, 0xa7, 0x6c, 0x00, 0x7a, 0x66, - 0xda, 0x03, 0xe2, 0xb9, 0x9e, 0x69, 0x07, 0x21, 0x33, 0xbf, 0xc9, 0xc2, 0x7a, 0x82, 0x2c, 0x98, - 0x79, 0x03, 0x10, 0xc9, 0x91, 0xb2, 0x92, 0xdd, 0x2a, 0xef, 0x7c, 0x9d, 0x92, 0x95, 0x19, 0x78, - 0x8d, 0x66, 0x04, 0xd6, 0xb6, 0x03, 0xef, 0x0a, 0xc7, 0xd0, 0xd1, 0x37, 0x50, 0x18, 0x12, 0x6d, - 0x14, 0x0c, 0x6b, 0x99, 0x3b, 0xd2, 0x56, 0x75, 0xe7, 0xd9, 0x0d, 0xd6, 0x39, 0x60, 0x40, 0xbd, - 0x40, 0x0b, 0x08, 0x16, 0xa8, 0xe8, 0x2e, 0x20, 0xfe, 0xa6, 0x1a, 0xc4, 0xd7, 0x3d, 0xd3, 0xa5, - 0x86, 0x5c, 0xcb, 0xde, 0x91, 0xb6, 0x4a, 0x78, 0x8d, 0xf7, 0xec, 0x4d, 0x3a, 0xea, 0x2e, 0xac, - 0x5e, 0xe3, 0x16, 0xc9, 0x90, 0x3d, 0x27, 0x57, 0x4c, 0x23, 0x25, 0x4c, 0x5f, 0xd1, 0x3e, 0xe4, - 0x2f, 0xb4, 0xd1, 0x98, 0x30, 0x96, 0xcb, 0x3b, 0x5f, 0x7e, 0xc8, 0x3c, 0x84, 0x89, 0x4e, 0xe4, - 0x80, 0xf9, 0xfc, 0x87, 0x99, 0x07, 0x92, 0xb2, 0x0b, 0xe5, 0x18, 0xdf, 0xa8, 0x0a, 0x70, 0xda, - 0xd9, 0x6b, 0xf7, 0xdb, 0xad, 0x7e, 0x7b, 0x4f, 0x5e, 0x42, 0x2b, 0x50, 0x3a, 0xed, 0x1c, 0xb4, - 0x9b, 0x47, 0xfd, 0x83, 0x97, 0xb2, 0x84, 0xca, 0xb0, 0x1c, 0x36, 0x32, 0xca, 0x25, 0x20, 0x4c, - 0x74, 0xe7, 0x82, 0x78, 0xd4, 0x90, 0x85, 0x56, 0xd1, 0x27, 0xb0, 0x1c, 0x68, 0xfe, 0xb9, 0x6a, - 0x1a, 0x82, 0xe7, 0x02, 0x6d, 0x1e, 0x1a, 0xe8, 0x10, 0x0a, 0x43, 0xcd, 0x36, 0x46, 0x1f, 0xe6, - 0x3b, 0x29, 0x6a, 0x0a, 0x7e, 0xc0, 0x26, 0x62, 0x01, 0x40, 0xad, 0x3b, 0xb1, 0x32, 0x57, 0x80, - 0xf2, 0x12, 0xe4, 0x5e, 0xa0, 0x79, 0x41, 0x9c, 0x9d, 0x36, 0xe4, 0xe8, 0xfa, 0xc2, 0xa2, 0xe7, - 0x59, 0x93, 0xef, 0x4c, 0xcc, 0xa6, 0x2b, 0xff, 0x9f, 0x81, 0xb5, 0x18, 0xb6, 0xb0, 0xd4, 0x17, - 0x50, 0xf0, 0x88, 0x3f, 0x1e, 0x05, 0x0c, 0xbe, 0xba, 0xf3, 0x24, 0x25, 0xfc, 0x14, 0x52, 0x03, - 0x33, 0x18, 0x2c, 0xe0, 0xd0, 0x16, 0xc8, 0x7c, 0x86, 0x4a, 0x3c, 0xcf, 0xf1, 0x54, 0xcb, 0x1f, - 0x30, 0xa9, 0x95, 0x70, 0x95, 0xd3, 0xdb, 0x94, 0x7c, 0xec, 0x0f, 0x62, 0x52, 0xcd, 0xde, 0x50, - 0xaa, 0x48, 0x03, 0xd9, 0x26, 0xc1, 0x3b, 0xc7, 0x3b, 0x57, 0xa9, 0x68, 0x3d, 0xd3, 0x20, 0xb5, - 0x1c, 0x03, 0xbd, 0x97, 0x12, 0xb4, 0xc3, 0xa7, 0x77, 0xc5, 0x6c, 0xbc, 0x6a, 0x27, 0x09, 0xca, - 0x0f, 0xa1, 0xc0, 0xbf, 0x94, 0x5a, 0x52, 0xef, 0xb4, 0xd5, 0x6a, 0xf7, 0x7a, 0xf2, 0x12, 0x2a, - 0x41, 0x1e, 0xb7, 0xfb, 0x98, 0x5a, 0x58, 0x09, 0xf2, 0xcf, 0x9a, 0xfd, 0xe6, 0x91, 0x9c, 0x51, - 0x7e, 0x00, 0xab, 0x2f, 0x34, 0x33, 0x48, 0x63, 0x5c, 0x8a, 0x03, 0xf2, 0x64, 0xac, 0xd0, 0xce, - 0x61, 0x42, 0x3b, 0xe9, 0x45, 0xd3, 0xbe, 0x34, 0x83, 0x6b, 0xfa, 0x90, 0x21, 0x4b, 0x3c, 0x4f, - 0xa8, 0x80, 0xbe, 0x2a, 0xef, 0x60, 0xb5, 0x17, 0x38, 0x6e, 0x2a, 0xcb, 0xff, 0x11, 0x2c, 0x53, - 0x1f, 0xe5, 0x8c, 0x03, 0x61, 0xfa, 0xb7, 0x1b, 0xdc, 0x87, 0x35, 0x42, 0x1f, 0xd6, 0xd8, 0x13, - 0x3e, 0x0e, 0x87, 0x23, 0xd1, 0x2d, 0x28, 0xf8, 0xe6, 0xc0, 0xd6, 0x46, 0xe2, 0xb4, 0x10, 0x2d, - 0x05, 0x51, 0x23, 0x0f, 0x17, 0x16, 0x86, 0xdf, 0x02, 0xb4, 0x47, 0xfc, 0xc0, 0x73, 0xae, 0x52, - 0xf1, 0xb3, 0x01, 0xf9, 0xd7, 0x8e, 0xa7, 0xf3, 0x8d, 0x58, 0xc4, 0xbc, 0x41, 0x37, 0x55, 0x02, - 0x44, 0x60, 0xdf, 0x05, 0x74, 0x68, 0x53, 0x9f, 0x92, 0x4e, 0x11, 0x7f, 0x9f, 0x81, 0xf5, 0xc4, - 0x78, 0xa1, 0x8c, 0xc5, 0xf7, 0x21, 0x3d, 0x98, 0xc6, 0x3e, 0xdf, 0x87, 0xa8, 0x0b, 0x05, 0x3e, - 0x42, 0x48, 0xf2, 0xfe, 0x1c, 0x40, 0xdc, 0x4d, 0x09, 0x38, 0x01, 0x33, 0xd3, 0xe8, 0xb3, 0x1f, - 0xd7, 0xe8, 0xdf, 0x81, 0x1c, 0x7e, 0x87, 0xff, 0x41, 0xdd, 0x7c, 0x0d, 0xeb, 0xba, 0x33, 0x1a, - 0x11, 0x9d, 0x5a, 0x83, 0x6a, 0xda, 0x01, 0xf1, 0x2e, 0xb4, 0xd1, 0x87, 0xed, 0x06, 0x4d, 0x66, - 0x1d, 0x8a, 0x49, 0xca, 0x2b, 0x58, 0x8b, 0x2d, 0x2c, 0x14, 0xf1, 0x0c, 0xf2, 0x3e, 0x25, 0x08, - 0x4d, 0x7c, 0x31, 0xa7, 0x26, 0x7c, 0xcc, 0xa7, 0x2b, 0xeb, 0x1c, 0xbc, 0x7d, 0x41, 0xec, 0xe8, - 0xb3, 0x94, 0x3d, 0x58, 0xeb, 0x31, 0x33, 0x4d, 0x65, 0x87, 0x13, 0x13, 0xcf, 0x24, 0x4c, 0x7c, - 0x03, 0x50, 0x1c, 0x45, 0x18, 0xe2, 0x15, 0xac, 0xb6, 0x2f, 0x89, 0x9e, 0x0a, 0xb9, 0x06, 0xcb, - 0xba, 0x63, 0x59, 0x9a, 0x6d, 0xd4, 0x32, 0x77, 0xb2, 0x5b, 0x25, 0x1c, 0x36, 0xe3, 0x7b, 0x31, - 0x9b, 0x76, 0x2f, 0x2a, 0x7f, 0x2b, 0x81, 0x3c, 0x59, 0x5b, 0x08, 0x92, 0x72, 0x1f, 0x18, 0x14, - 0x88, 0xae, 0x5d, 0xc1, 0xa2, 0x25, 0xe8, 0xe1, 0x71, 0xc1, 0xe9, 0xc4, 0xf3, 0x62, 0xc7, 0x51, - 0xf6, 0x86, 0xc7, 0x91, 0x72, 0x00, 0xbf, 0x13, 0xb2, 0xd3, 0x0b, 0x3c, 0xa2, 0x59, 0xa6, 0x3d, - 0x38, 0xec, 0x76, 0x5d, 0xc2, 0x19, 0x47, 0x08, 0x72, 0x86, 0x16, 0x68, 0x82, 0x31, 0xf6, 0x4e, - 0x37, 0xbd, 0x3e, 0x72, 0xfc, 0x68, 0xd3, 0xb3, 0x86, 0xf2, 0x1f, 0x59, 0xa8, 0x4d, 0x41, 0x85, - 0xe2, 0x7d, 0x05, 0x79, 0x9f, 0x04, 0x63, 0x57, 0x98, 0x4a, 0x3b, 0x35, 0xc3, 0xb3, 0xf1, 0x1a, - 0x3d, 0x0a, 0x86, 0x39, 0x26, 0x1a, 0x40, 0x31, 0x08, 0xae, 0x54, 0xdf, 0xfc, 0x79, 0x18, 0x10, - 0x1c, 0xdd, 0x14, 0xbf, 0x4f, 0x3c, 0xcb, 0xb4, 0xb5, 0x51, 0xcf, 0xfc, 0x39, 0xc1, 0xcb, 0x41, - 0x70, 0x45, 0x5f, 0xd0, 0x4b, 0x6a, 0xf0, 0x86, 0x69, 0x0b, 0xb1, 0xb7, 0x16, 0x5d, 0x25, 0x26, - 0x60, 0xcc, 0x11, 0xeb, 0x47, 0x90, 0x67, 0xdf, 0xb4, 0x88, 0x21, 0xca, 0x90, 0x0d, 0x82, 0x2b, - 0xc6, 0x54, 0x11, 0xd3, 0xd7, 0xfa, 0x23, 0xa8, 0xc4, 0xbf, 0x80, 0x1a, 0xd2, 0x90, 0x98, 0x83, - 0x21, 0x37, 0xb0, 0x3c, 0x16, 0x2d, 0xaa, 0xc9, 0x77, 0xa6, 0x21, 0x42, 0xd6, 0x3c, 0xe6, 0x0d, - 0xe5, 0x5f, 0x33, 0x70, 0x7b, 0x86, 0x64, 0x84, 0xb1, 0xbe, 0x4a, 0x18, 0xeb, 0x47, 0x92, 0x42, - 0x68, 0xf1, 0xaf, 0x12, 0x16, 0xff, 0x11, 0xc1, 0xe9, 0xb6, 0xb9, 0x05, 0x05, 0x72, 0x69, 0x06, - 0xc4, 0x10, 0xa2, 0x12, 0xad, 0xd8, 0x76, 0xca, 0xdd, 0x74, 0x3b, 0x7d, 0x09, 0x1b, 0x2d, 0x8f, - 0x68, 0x01, 0x11, 0x47, 0x79, 0x68, 0xff, 0xb7, 0xa1, 0xa8, 0x8d, 0x46, 0x8e, 0x3e, 0x51, 0xeb, - 0x32, 0x6b, 0x1f, 0x1a, 0xca, 0xb7, 0x12, 0x6c, 0x5e, 0x9b, 0x23, 0x24, 0x7d, 0x06, 0x55, 0xd3, - 0x77, 0x46, 0xec, 0x23, 0xd4, 0xd8, 0x2d, 0xee, 0xc7, 0xf3, 0xb9, 0x93, 0xc3, 0x10, 0x83, 0x5d, - 0xea, 0x56, 0xcc, 0x78, 0x93, 0x59, 0x15, 0x5b, 0xdc, 0x10, 0xbb, 0x39, 0x6c, 0x2a, 0xff, 0x20, - 0xc1, 0xa6, 0xf0, 0xe2, 0xa9, 0x3f, 0x66, 0x06, 0xcb, 0x99, 0x8f, 0xcd, 0xb2, 0x52, 0x83, 0x5b, - 0xd7, 0xf9, 0x12, 0xe7, 0xfa, 0x7f, 0xe6, 0x00, 0x4d, 0xdf, 0x20, 0xd1, 0xf7, 0xa0, 0xe2, 0x13, - 0xdb, 0x50, 0xb9, 0x4f, 0xe0, 0xee, 0xaa, 0x88, 0xcb, 0x94, 0xc6, 0x9d, 0x83, 0x4f, 0x8f, 0x39, - 0x72, 0x29, 0xb8, 0x2d, 0x62, 0xf6, 0x8e, 0x86, 0x50, 0x79, 0xed, 0xab, 0xd1, 0xda, 0xcc, 0x68, - 0xaa, 0xa9, 0x8f, 0xae, 0x69, 0x3e, 0x1a, 0xcf, 0x7a, 0xd1, 0x77, 0xe1, 0xf2, 0x6b, 0x3f, 0x6a, - 0xa0, 0x5f, 0x4a, 0xf0, 0x49, 0x18, 0x3a, 0x4c, 0xc4, 0x67, 0x39, 0x06, 0xf1, 0x6b, 0xb9, 0x3b, - 0xd9, 0xad, 0xea, 0xce, 0xc9, 0x0d, 0xe4, 0x37, 0x45, 0x3c, 0x76, 0x0c, 0x82, 0x37, 0xed, 0x19, - 0x54, 0x1f, 0x35, 0x60, 0xdd, 0x1a, 0xfb, 0x81, 0xca, 0xad, 0x40, 0x15, 0x83, 0x6a, 0x79, 0x26, - 0x97, 0x35, 0xda, 0x95, 0xb0, 0x55, 0x74, 0x0e, 0x2b, 0x96, 0x33, 0xb6, 0x03, 0x55, 0x67, 0x77, - 0x1c, 0xbf, 0x56, 0x98, 0xeb, 0xf2, 0x3b, 0x43, 0x4a, 0xc7, 0x14, 0x8e, 0xdf, 0x98, 0x7c, 0x5c, - 0xb1, 0x62, 0x2d, 0xa5, 0x01, 0xe5, 0x98, 0x0c, 0x51, 0x11, 0x72, 0x9d, 0x6e, 0xa7, 0x2d, 0x2f, - 0x21, 0x80, 0x42, 0xeb, 0x00, 0x77, 0xbb, 0x7d, 0x1e, 0xf6, 0x1f, 0x1e, 0x37, 0xf7, 0xdb, 0x72, - 0x46, 0x69, 0x43, 0x25, 0x8e, 0x86, 0x10, 0x54, 0x4f, 0x3b, 0xcf, 0x3b, 0xdd, 0x17, 0x1d, 0xf5, - 0xb8, 0x7b, 0xda, 0xe9, 0xd3, 0x0b, 0x43, 0x15, 0xa0, 0xd9, 0x79, 0x39, 0x69, 0xaf, 0x40, 0xa9, - 0xd3, 0x0d, 0x9b, 0x52, 0x3d, 0x23, 0x4b, 0xca, 0xff, 0x65, 0x60, 0x63, 0x96, 0x60, 0x91, 0x01, - 0x39, 0xaa, 0x24, 0x71, 0x65, 0xfb, 0xf8, 0x3a, 0x62, 0xe8, 0xd4, 0x36, 0x5d, 0x4d, 0x9c, 0xd1, - 0x25, 0xcc, 0xde, 0x91, 0x0a, 0x85, 0x91, 0x76, 0x46, 0x46, 0x7e, 0x2d, 0xcb, 0x92, 0x1a, 0xfb, - 0x37, 0x59, 0xfb, 0x88, 0x21, 0xf1, 0x8c, 0x86, 0x80, 0xad, 0xef, 0x42, 0x39, 0x46, 0x9e, 0x91, - 0x3a, 0xd8, 0x88, 0xa7, 0x0e, 0x4a, 0xf1, 0x3c, 0xc0, 0x93, 0x69, 0x69, 0xd1, 0xaf, 0xa1, 0xea, - 0x3a, 0xe8, 0xf6, 0xfa, 0xfc, 0x92, 0xb6, 0x8f, 0xbb, 0xa7, 0x27, 0xb2, 0x44, 0x89, 0xfd, 0x66, - 0xef, 0xb9, 0x9c, 0x89, 0xb4, 0x99, 0x55, 0xfe, 0x65, 0x19, 0x60, 0x72, 0x6d, 0x46, 0x55, 0xc8, - 0x44, 0x07, 0x4d, 0xc6, 0x34, 0xa8, 0x3c, 0x6c, 0xcd, 0x0a, 0x17, 0x66, 0xef, 0x68, 0x07, 0x36, - 0x2d, 0x7f, 0xe0, 0x6a, 0xfa, 0xb9, 0x2a, 0x6e, 0xbb, 0xdc, 0x1e, 0xd9, 0xa6, 0xad, 0xe0, 0x75, - 0xd1, 0x29, 0xcc, 0x8d, 0xe3, 0x1e, 0x41, 0x96, 0xd8, 0x17, 0x6c, 0x83, 0x95, 0x77, 0x1e, 0xce, - 0x7d, 0x9d, 0x6f, 0xb4, 0xed, 0x0b, 0x2e, 0x33, 0x0a, 0x83, 0x54, 0x00, 0x83, 0x5c, 0x98, 0x3a, - 0x51, 0x29, 0x68, 0x9e, 0x81, 0x3e, 0x9d, 0x1f, 0x74, 0x8f, 0x61, 0x44, 0xd0, 0x25, 0x23, 0x6c, - 0xa3, 0x0e, 0x94, 0x3c, 0xe2, 0x3b, 0x63, 0x4f, 0x27, 0x7c, 0x97, 0xa5, 0x8f, 0xb8, 0x71, 0x38, - 0x0f, 0x4f, 0x20, 0xd0, 0x1e, 0x14, 0xd8, 0xe6, 0xf2, 0x6b, 0xcb, 0x8c, 0xd9, 0xcf, 0x53, 0x82, - 0xb1, 0x1d, 0x85, 0xc5, 0x5c, 0xb4, 0x0f, 0xcb, 0x9c, 0x45, 0xbf, 0x56, 0x64, 0x30, 0x77, 0xd3, - 0xee, 0x7c, 0x36, 0x0b, 0x87, 0xb3, 0xa9, 0x56, 0xc7, 0x3e, 0xf1, 0x6a, 0x25, 0xae, 0x55, 0xfa, - 0x8e, 0x3e, 0x85, 0x12, 0x77, 0x34, 0x86, 0xe9, 0xd5, 0x80, 0x75, 0x70, 0xcf, 0xb3, 0x67, 0x7a, - 0xe8, 0x33, 0x28, 0xf3, 0xa0, 0x41, 0x65, 0xbb, 0xa3, 0xcc, 0xba, 0x81, 0x93, 0x4e, 0xe8, 0x1e, - 0xe1, 0x03, 0x88, 0xe7, 0xf1, 0x01, 0x95, 0x68, 0x00, 0xf1, 0x3c, 0x36, 0xe0, 0xf7, 0x61, 0x95, - 0x85, 0x5a, 0x03, 0xcf, 0x19, 0xbb, 0x2a, 0xb3, 0xa9, 0x15, 0x36, 0x68, 0x85, 0x92, 0xf7, 0x29, - 0xb5, 0x43, 0x8d, 0xeb, 0x36, 0x14, 0xdf, 0x38, 0x67, 0x7c, 0x40, 0x95, 0xfb, 0xbb, 0x37, 0xce, - 0x59, 0xd8, 0x15, 0xb9, 0xc2, 0xd5, 0xa4, 0x2b, 0x7c, 0x0b, 0xb7, 0xa6, 0xcf, 0x74, 0xe6, 0x12, - 0xe5, 0x9b, 0xbb, 0xc4, 0x0d, 0x7b, 0x06, 0xb5, 0x7e, 0x0f, 0x8a, 0xa1, 0xe5, 0xcc, 0xb3, 0x63, - 0xeb, 0x8f, 0xa0, 0x9a, 0xb4, 0xbb, 0xb9, 0xf6, 0xfb, 0x7f, 0x49, 0x50, 0x8a, 0x2c, 0x0c, 0xd9, - 0xb0, 0xce, 0x24, 0x40, 0x63, 0x08, 0x75, 0x62, 0xb0, 0x3c, 0x72, 0x79, 0x9c, 0xf2, 0x9b, 0x9b, - 0x21, 0x82, 0xb8, 0x26, 0x09, 0xeb, 0x45, 0x11, 0xf2, 0x64, 0xbd, 0x6f, 0x60, 0x75, 0x64, 0xda, - 0xe3, 0xcb, 0xd8, 0x5a, 0x3c, 0xe4, 0xf8, 0x83, 0x94, 0x6b, 0x1d, 0xd1, 0xd9, 0x93, 0x35, 0xaa, - 0xa3, 0x44, 0x5b, 0xf9, 0x36, 0x03, 0xb7, 0x66, 0xb3, 0x83, 0x3a, 0x90, 0xd5, 0xdd, 0xb1, 0xf8, - 0xb4, 0x47, 0xf3, 0x7e, 0x5a, 0xcb, 0x1d, 0x4f, 0x56, 0xa5, 0x40, 0xe8, 0x05, 0x14, 0x2c, 0x62, - 0x39, 0xde, 0x95, 0xf8, 0x82, 0x27, 0xf3, 0x42, 0x1e, 0xb3, 0xd9, 0x13, 0x54, 0x01, 0x87, 0x30, - 0x14, 0x85, 0xbd, 0xf8, 0xe2, 0x64, 0x9a, 0x33, 0x23, 0x11, 0x42, 0xe2, 0x08, 0x47, 0xb9, 0x07, - 0x9b, 0x33, 0x3f, 0x05, 0xfd, 0x2e, 0x80, 0xee, 0x8e, 0x55, 0x96, 0x31, 0xe6, 0x7a, 0xcf, 0xe2, - 0x92, 0xee, 0x8e, 0x7b, 0x8c, 0xa0, 0xdc, 0x87, 0xda, 0xfb, 0xf8, 0xa5, 0xfb, 0x9d, 0x73, 0xac, - 0x5a, 0x67, 0x4c, 0x06, 0x59, 0x5c, 0xe4, 0x84, 0xe3, 0x33, 0xe5, 0x57, 0x19, 0x58, 0xbd, 0xc6, - 0x0e, 0x8d, 0xe8, 0xf9, 0xf9, 0x11, 0xde, 0x95, 0x78, 0x8b, 0x1e, 0x26, 0xba, 0x69, 0x84, 0x59, - 0x36, 0xf6, 0xce, 0xdc, 0x88, 0x2b, 0x32, 0x60, 0x19, 0xd3, 0xa5, 0x06, 0x6d, 0x9d, 0x99, 0x81, - 0xcf, 0x82, 0xfe, 0x3c, 0xe6, 0x0d, 0xf4, 0x12, 0xaa, 0x1e, 0xf1, 0x89, 0x77, 0x41, 0x0c, 0xd5, - 0x75, 0xbc, 0x20, 0x14, 0xd8, 0xce, 0x7c, 0x02, 0x3b, 0x71, 0xbc, 0x00, 0xaf, 0x84, 0x48, 0xb4, - 0xe5, 0xa3, 0x17, 0xb0, 0x62, 0x5c, 0xd9, 0x9a, 0x65, 0xea, 0x02, 0xb9, 0xb0, 0x30, 0x72, 0x45, - 0x00, 0x31, 0x60, 0x65, 0x17, 0xca, 0xb1, 0x4e, 0xfa, 0x61, 0xcc, 0x89, 0x0b, 0x99, 0xf0, 0x46, - 0x72, 0xff, 0xe6, 0xc5, 0xfe, 0x55, 0xfe, 0x29, 0x03, 0xd5, 0xe4, 0x06, 0x08, 0xf5, 0xe7, 0x12, - 0xcf, 0x74, 0x8c, 0x98, 0xfe, 0x4e, 0x18, 0x81, 0xea, 0x88, 0x76, 0xbf, 0x1d, 0x3b, 0x81, 0x16, - 0xea, 0x48, 0x77, 0xc7, 0x7f, 0x48, 0xdb, 0xd7, 0x74, 0x9f, 0xbd, 0xa6, 0x7b, 0xf4, 0x39, 0x20, - 0xa1, 0xdf, 0x91, 0x69, 0x99, 0x81, 0x7a, 0x76, 0x15, 0x10, 0x2e, 0xff, 0x2c, 0x96, 0x79, 0xcf, - 0x11, 0xed, 0xf8, 0x8a, 0xd2, 0x91, 0x02, 0x2b, 0x8e, 0x63, 0xa9, 0xbe, 0xee, 0x78, 0x44, 0xd5, - 0x8c, 0x37, 0x2c, 0x08, 0xcd, 0xe2, 0xb2, 0xe3, 0x58, 0x3d, 0x4a, 0x6b, 0x1a, 0x6f, 0xe8, 0x19, - 0xaf, 0xbb, 0x63, 0x9f, 0x04, 0x2a, 0x7d, 0x30, 0xb7, 0x58, 0xc2, 0xc0, 0x49, 0x2d, 0x77, 0xec, - 0xc7, 0x06, 0x58, 0xc4, 0xa2, 0xae, 0x2e, 0x36, 0xe0, 0x98, 0x58, 0x74, 0x95, 0xca, 0x09, 0xf1, - 0x74, 0x62, 0x07, 0x7d, 0x53, 0x3f, 0xa7, 0x5e, 0x4c, 0xda, 0x92, 0x70, 0x82, 0xa6, 0xfc, 0x0c, - 0xf2, 0xcc, 0xeb, 0xd1, 0x8f, 0x67, 0x1e, 0x83, 0x39, 0x14, 0x2e, 0xde, 0x22, 0x25, 0x30, 0x77, - 0xf2, 0x29, 0x94, 0x86, 0x8e, 0x2f, 0xdc, 0x11, 0xb7, 0xbc, 0x22, 0x25, 0xb0, 0xce, 0x3a, 0x14, - 0x3d, 0xa2, 0x19, 0x8e, 0x3d, 0x0a, 0x2f, 0xea, 0x51, 0x5b, 0x79, 0x0b, 0x05, 0x7e, 0xfc, 0xde, - 0x00, 0xff, 0x2e, 0x20, 0x9d, 0xfb, 0x31, 0x97, 0x5e, 0xfc, 0x7d, 0xdf, 0x74, 0x6c, 0x3f, 0xfc, - 0x3b, 0xc4, 0x7b, 0x4e, 0x26, 0x1d, 0xca, 0x7f, 0x4b, 0x3c, 0xc4, 0xe2, 0x79, 0x7b, 0x7a, 0x0b, - 0xa4, 0x96, 0x46, 0x6f, 0x39, 0x3c, 0x41, 0x10, 0x36, 0xe9, 0xdd, 0x58, 0x44, 0x52, 0x99, 0x45, - 0x7f, 0x7b, 0x08, 0x80, 0x30, 0x5d, 0x48, 0xc4, 0x45, 0x6a, 0xde, 0x74, 0x21, 0xe1, 0xe9, 0x42, - 0x42, 0xaf, 0x73, 0x22, 0xc6, 0xe3, 0x70, 0x39, 0x16, 0xe2, 0x95, 0x8d, 0x28, 0x27, 0x4b, 0x94, - 0xff, 0x95, 0xa2, 0xb3, 0x22, 0xcc, 0x9d, 0xa2, 0x6f, 0xa0, 0x48, 0xb7, 0x9d, 0x6a, 0x69, 0xae, - 0xf8, 0x13, 0xd8, 0x5a, 0x2c, 0x2d, 0xdb, 0xa0, 0xbb, 0xec, 0x58, 0x73, 0x79, 0x84, 0xb6, 0xec, - 0xf2, 0x16, 0x3d, 0x73, 0x34, 0x63, 0x72, 0xe6, 0xd0, 0x77, 0xf4, 0x7d, 0xa8, 0x6a, 0xe3, 0xc0, - 0x51, 0x35, 0xe3, 0x82, 0x78, 0x81, 0xe9, 0x13, 0xa1, 0xfb, 0x15, 0x4a, 0x6d, 0x86, 0xc4, 0xfa, - 0x43, 0xa8, 0xc4, 0x31, 0x3f, 0xe4, 0x7d, 0xf3, 0x71, 0xef, 0xfb, 0x27, 0x00, 0x93, 0x3c, 0x04, - 0xb5, 0x11, 0x72, 0x69, 0xd2, 0xdb, 0x98, 0xb8, 0x96, 0xe4, 0x71, 0x91, 0x12, 0x5a, 0x34, 0x00, - 0x4f, 0x26, 0x49, 0xf3, 0x61, 0x92, 0x94, 0xee, 0x5a, 0xba, 0xd1, 0xce, 0xcd, 0xd1, 0x28, 0xca, - 0x8d, 0x94, 0x1c, 0xc7, 0x7a, 0xce, 0x08, 0xca, 0x6f, 0x32, 0xdc, 0x56, 0x78, 0xba, 0x3b, 0x55, - 0x38, 0xfe, 0xb1, 0x54, 0xbd, 0x0b, 0xe0, 0x07, 0x9a, 0x47, 0x43, 0x09, 0x2d, 0xcc, 0xce, 0xd4, - 0xa7, 0xb2, 0xac, 0xfd, 0xf0, 0xaf, 0x3d, 0x2e, 0x89, 0xd1, 0xcd, 0x00, 0x3d, 0x86, 0x8a, 0xee, - 0x58, 0xee, 0x88, 0x88, 0xc9, 0xf9, 0x0f, 0x4e, 0x2e, 0x47, 0xe3, 0x9b, 0x41, 0x2c, 0x27, 0x54, - 0xb8, 0x69, 0x4e, 0xe8, 0xdf, 0x24, 0x9e, 0xb5, 0x8f, 0xff, 0x34, 0x40, 0x83, 0x19, 0x7f, 0xa6, - 0xf7, 0x17, 0xfc, 0x03, 0xf1, 0x5d, 0xbf, 0xa5, 0xeb, 0x8f, 0xd3, 0xfc, 0x07, 0x7e, 0x7f, 0x70, - 0xf7, 0xef, 0x59, 0x28, 0x45, 0x09, 0xfb, 0x29, 0xdd, 0x3f, 0x80, 0x52, 0x54, 0x32, 0x21, 0x0e, - 0x88, 0xef, 0x54, 0x4f, 0x34, 0x18, 0xbd, 0x06, 0xa4, 0x0d, 0x06, 0x51, 0xd0, 0xa6, 0x8e, 0x7d, - 0x6d, 0x10, 0xfe, 0x2e, 0x79, 0x30, 0x87, 0x1c, 0x42, 0xbf, 0x75, 0x4a, 0xe7, 0x63, 0x59, 0x1b, - 0x0c, 0x12, 0x14, 0xf4, 0xa7, 0xb0, 0x99, 0x5c, 0x43, 0x3d, 0xbb, 0x52, 0x5d, 0xd3, 0x10, 0xd7, - 0xbe, 0x83, 0x79, 0xff, 0x59, 0x34, 0x12, 0xf0, 0x5f, 0x5d, 0x9d, 0x98, 0x06, 0x97, 0x39, 0xf2, - 0xa6, 0x3a, 0xea, 0x7f, 0x0e, 0x9f, 0xbc, 0x67, 0xf8, 0x0c, 0x1d, 0x74, 0x92, 0xff, 0xe2, 0x17, - 0x17, 0x42, 0x4c, 0x7b, 0xbf, 0x96, 0xf8, 0xaf, 0x95, 0xa4, 0x4c, 0x9a, 0xf1, 0xb8, 0x75, 0x3b, - 0xe5, 0x3a, 0xad, 0x93, 0x53, 0x0e, 0xcf, 0x42, 0xd5, 0xaf, 0xaf, 0x85, 0xaa, 0x69, 0x83, 0x18, - 0x1e, 0xf1, 0x71, 0x20, 0x81, 0xa0, 0xfc, 0x73, 0x16, 0x8a, 0x21, 0x3a, 0xbb, 0xb4, 0x5d, 0xf9, - 0x01, 0xb1, 0xd4, 0x28, 0xb3, 0x22, 0x61, 0xe0, 0x24, 0x96, 0x45, 0xf8, 0x14, 0x4a, 0xf4, 0x6e, - 0xc8, 0xbb, 0x33, 0xac, 0xbb, 0x48, 0x09, 0xac, 0xf3, 0x33, 0x28, 0x07, 0x4e, 0xa0, 0x8d, 0xd4, - 0x80, 0xf9, 0xf2, 0x2c, 0x9f, 0xcd, 0x48, 0xcc, 0x93, 0xa3, 0x1f, 0xc2, 0x5a, 0x30, 0xf4, 0x9c, - 0x20, 0x18, 0xd1, 0xf8, 0x8e, 0x45, 0x34, 0x3c, 0x00, 0xc9, 0x61, 0x39, 0xea, 0xe0, 0x91, 0x8e, - 0x4f, 0x4f, 0xef, 0xc9, 0x60, 0x6a, 0xba, 0xec, 0x10, 0xc9, 0xe1, 0x95, 0x88, 0x4a, 0x4d, 0x9b, - 0x3a, 0x4f, 0x97, 0x47, 0x0b, 0xec, 0xac, 0x90, 0x70, 0xd8, 0x44, 0x2a, 0xac, 0x5a, 0x44, 0xf3, - 0xc7, 0x1e, 0x31, 0xd4, 0xd7, 0x26, 0x19, 0x19, 0xfc, 0xae, 0x5d, 0x4d, 0x1d, 0x7e, 0x87, 0x62, - 0x69, 0x3c, 0x63, 0xb3, 0x71, 0x35, 0x84, 0xe3, 0x6d, 0x1a, 0x39, 0xf0, 0x37, 0xb4, 0x0a, 0xe5, - 0xde, 0xcb, 0x5e, 0xbf, 0x7d, 0xac, 0x1e, 0x77, 0xf7, 0xda, 0xa2, 0xdc, 0xa2, 0xd7, 0xc6, 0xbc, - 0x29, 0xd1, 0xfe, 0x7e, 0xb7, 0xdf, 0x3c, 0x52, 0xfb, 0x87, 0xad, 0xe7, 0x3d, 0x39, 0x83, 0x36, - 0x61, 0xad, 0x7f, 0x80, 0xbb, 0xfd, 0xfe, 0x51, 0x7b, 0x4f, 0x3d, 0x69, 0xe3, 0xc3, 0xee, 0x5e, - 0x4f, 0xce, 0x22, 0x04, 0xd5, 0x09, 0xb9, 0x7f, 0x78, 0xdc, 0x96, 0x73, 0xa8, 0x0c, 0xcb, 0x27, - 0x6d, 0xdc, 0x6a, 0x77, 0xfa, 0x72, 0x5e, 0xf9, 0x55, 0x16, 0xca, 0x31, 0x2d, 0x52, 0x43, 0xf6, - 0x7c, 0x1e, 0xe7, 0xe7, 0x30, 0x7d, 0x65, 0xbf, 0x87, 0x34, 0x7d, 0xc8, 0xb5, 0x93, 0xc3, 0xbc, - 0xc1, 0x62, 0x7b, 0xed, 0x32, 0xb6, 0xcf, 0x73, 0xb8, 0x68, 0x69, 0x97, 0x1c, 0xe4, 0x7b, 0x50, - 0x39, 0x27, 0x9e, 0x4d, 0x46, 0xa2, 0x9f, 0x6b, 0xa4, 0xcc, 0x69, 0x7c, 0xc8, 0x16, 0xc8, 0x62, - 0xc8, 0x04, 0x86, 0xab, 0xa3, 0xca, 0xe9, 0xc7, 0x21, 0xd8, 0x06, 0xe4, 0x79, 0xf7, 0x32, 0x5f, - 0x9f, 0x35, 0xa8, 0x9b, 0xf2, 0xdf, 0x69, 0x2e, 0x8b, 0xef, 0x72, 0x98, 0xbd, 0xa3, 0xb3, 0x69, - 0xfd, 0x14, 0x98, 0x7e, 0x76, 0xe7, 0x37, 0xe7, 0xf7, 0xa9, 0x68, 0x18, 0xa9, 0x68, 0x19, 0xb2, - 0x38, 0xac, 0x51, 0x68, 0x35, 0x5b, 0x07, 0x54, 0x2d, 0x2b, 0x50, 0x3a, 0x6e, 0xfe, 0x54, 0x3d, - 0xed, 0xb1, 0x84, 0x25, 0x92, 0xa1, 0xf2, 0xbc, 0x8d, 0x3b, 0xed, 0x23, 0x41, 0xc9, 0xa2, 0x0d, - 0x90, 0x05, 0x65, 0x32, 0x2e, 0x47, 0x11, 0xf8, 0x6b, 0x1e, 0x15, 0x21, 0xd7, 0x7b, 0xd1, 0x3c, - 0x91, 0x0b, 0xca, 0xff, 0x64, 0x60, 0x95, 0xbb, 0x85, 0xe8, 0x6f, 0xea, 0xfb, 0xff, 0x26, 0xc5, - 0x13, 0x17, 0x99, 0x64, 0xe2, 0x22, 0x0c, 0x42, 0x99, 0x57, 0xcf, 0x4e, 0x82, 0x50, 0x96, 0xf0, - 0x48, 0x9c, 0xf8, 0xb9, 0x79, 0x4e, 0xfc, 0x1a, 0x2c, 0x5b, 0xc4, 0x8f, 0xf4, 0x56, 0xc2, 0x61, - 0x13, 0x99, 0x50, 0xd6, 0x6c, 0xdb, 0x09, 0x58, 0x22, 0x23, 0xbc, 0x16, 0xed, 0xcf, 0x95, 0x41, - 0x8e, 0xbe, 0xb8, 0xd1, 0x9c, 0x20, 0xf1, 0x83, 0x39, 0x8e, 0x5d, 0xff, 0x09, 0xc8, 0xd7, 0x07, - 0xcc, 0xe3, 0x0e, 0x7f, 0xf0, 0xe5, 0xc4, 0x1b, 0x12, 0xba, 0x2f, 0x44, 0x3a, 0x59, 0x5e, 0xa2, - 0x0d, 0x7c, 0xda, 0xe9, 0x1c, 0x76, 0xf6, 0x65, 0x09, 0x01, 0x14, 0xda, 0x3f, 0x3d, 0xec, 0xb7, - 0xf7, 0xe4, 0xcc, 0xce, 0xaf, 0xd7, 0xa0, 0xc0, 0x99, 0x44, 0xdf, 0x8a, 0x48, 0x20, 0x5e, 0xa9, - 0x87, 0x7e, 0x32, 0x77, 0x44, 0x9d, 0xa8, 0xfe, 0xab, 0x3f, 0x59, 0x78, 0xbe, 0xf8, 0x6b, 0xb2, - 0x84, 0xfe, 0x5a, 0x82, 0x4a, 0xe2, 0x8f, 0x49, 0xda, 0x6c, 0xe8, 0x8c, 0xc2, 0xc0, 0xfa, 0x8f, - 0x17, 0x9a, 0x1b, 0xf1, 0xf2, 0x4b, 0x09, 0xca, 0xb1, 0x92, 0x38, 0xb4, 0xbb, 0x48, 0x19, 0x1d, - 0xe7, 0xe4, 0xe1, 0xe2, 0x15, 0x78, 0xca, 0xd2, 0x17, 0x12, 0xfa, 0x2b, 0x09, 0xca, 0xb1, 0xe2, - 0xb0, 0xd4, 0xac, 0x4c, 0x97, 0xb2, 0xa5, 0x66, 0x65, 0x56, 0x2d, 0xda, 0x12, 0xfa, 0x0b, 0x09, - 0x4a, 0x51, 0xa1, 0x17, 0xba, 0x3f, 0x7f, 0x69, 0x18, 0x67, 0xe2, 0xc1, 0xa2, 0x35, 0x65, 0xca, - 0x12, 0xfa, 0x33, 0x28, 0x86, 0x55, 0x51, 0x28, 0xad, 0xf7, 0xba, 0x56, 0x72, 0x55, 0xbf, 0x3f, - 0xf7, 0xbc, 0xf8, 0xf2, 0x61, 0xa9, 0x52, 0xea, 0xe5, 0xaf, 0x15, 0x55, 0xd5, 0xef, 0xcf, 0x3d, - 0x2f, 0x5a, 0x9e, 0x5a, 0x42, 0xac, 0xa2, 0x29, 0xb5, 0x25, 0x4c, 0x97, 0x52, 0xa5, 0xb6, 0x84, - 0x59, 0x05, 0x54, 0x9c, 0x91, 0x58, 0x4d, 0x54, 0x6a, 0x46, 0xa6, 0xeb, 0xae, 0x52, 0x33, 0x32, - 0xa3, 0x04, 0x4b, 0x59, 0x42, 0xbf, 0x90, 0xe2, 0xf7, 0x82, 0xfb, 0x73, 0x97, 0xfe, 0xcc, 0x69, - 0x92, 0x53, 0xc5, 0x47, 0x6c, 0x83, 0xfe, 0x42, 0x64, 0x31, 0x78, 0xe5, 0x10, 0x9a, 0x07, 0x2c, - 0x51, 0x6c, 0x54, 0xbf, 0xb7, 0x98, 0xb3, 0x61, 0x4c, 0xfc, 0xa5, 0x04, 0x30, 0xa9, 0x31, 0x4a, - 0xcd, 0xc4, 0x54, 0x71, 0x53, 0x7d, 0x77, 0x81, 0x99, 0xf1, 0x0d, 0x12, 0xd6, 0x40, 0xa4, 0xde, - 0x20, 0xd7, 0x6a, 0xa0, 0x52, 0x6f, 0x90, 0xeb, 0xf5, 0x4b, 0xca, 0x12, 0xfa, 0x47, 0x09, 0xd6, - 0xa6, 0x6a, 0x30, 0xd0, 0x93, 0x1b, 0x96, 0xe1, 0xd4, 0x9f, 0x2e, 0x0e, 0x10, 0xb2, 0xb6, 0x25, - 0x7d, 0x21, 0xa1, 0xbf, 0x91, 0x60, 0x25, 0xf9, 0xdf, 0x3a, 0xb5, 0x97, 0x9a, 0x51, 0xcd, 0x51, - 0x7f, 0xb4, 0xd8, 0xe4, 0x48, 0x5a, 0x7f, 0x27, 0x41, 0x35, 0x59, 0xc2, 0x80, 0x1e, 0xcd, 0x77, - 0x2c, 0x5c, 0x63, 0xe8, 0xf1, 0x82, 0xb3, 0x43, 0x8e, 0xbe, 0x5a, 0xfe, 0xa3, 0x3c, 0x8f, 0xde, - 0x0a, 0xec, 0xf1, 0xa3, 0xdf, 0x06, 0x00, 0x00, 0xff, 0xff, 0x53, 0x00, 0xc0, 0x95, 0x86, 0x31, - 0x00, 0x00, + // 3643 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x5a, 0x4f, 0x6f, 0x1b, 0x49, + 0x76, 0x57, 0xf3, 0x9f, 0xc8, 0x47, 0x89, 0x6a, 0x95, 0x65, 0x0f, 0xcd, 0x49, 0x32, 0xde, 0x06, + 0x36, 0x10, 0x76, 0x67, 0xe8, 0x19, 0x2d, 0x32, 0x1e, 0xcd, 0x7a, 0xd6, 0xc3, 0xa1, 0x68, 0x49, + 0x63, 0x89, 0x52, 0x8a, 0x14, 0xbc, 0x8e, 0xb3, 0xd3, 0x69, 0x75, 0x97, 0xc9, 0xb6, 0xd8, 0x7f, + 0xdc, 0xd5, 0x94, 0xa5, 0x0d, 0x82, 0x04, 0x1b, 0x20, 0xd8, 0x00, 0x09, 0x92, 0xcb, 0x64, 0x2f, + 0x39, 0x6d, 0x8e, 0xf9, 0x02, 0x41, 0x82, 0x3d, 0xe7, 0x0b, 0xe4, 0x96, 0x5c, 0x72, 0xcb, 0x25, + 0x40, 0xf2, 0x0d, 0x16, 0xf5, 0xa7, 0x9b, 0xdd, 0x24, 0x3d, 0x6e, 0x52, 0x3e, 0x91, 0xef, 0x55, + 0xd5, 0xaf, 0x5e, 0xbd, 0xf7, 0xaa, 0xde, 0xab, 0xea, 0x07, 0x9a, 0x3f, 0x1a, 0x0f, 0x6c, 0x97, + 0xde, 0xb7, 0x02, 0xfb, 0x92, 0x04, 0xf4, 0xbe, 0x1f, 0x78, 0xa1, 0x27, 0xa9, 0x26, 0x27, 0xd0, + 0xf7, 0x87, 0x06, 0x1d, 0xda, 0xa6, 0x17, 0xf8, 0x4d, 0xd7, 0x73, 0x0c, 0xab, 0x29, 0xc7, 0x34, + 0xe5, 0x18, 0xd1, 0xad, 0xf1, 0x7b, 0x03, 0xcf, 0x1b, 0x8c, 0x88, 0x40, 0x38, 0x1f, 0xbf, 0xb8, + 0x6f, 0x8d, 0x03, 0x23, 0xb4, 0x3d, 0x57, 0xb6, 0x7f, 0x30, 0xdd, 0x1e, 0xda, 0x0e, 0xa1, 0xa1, + 0xe1, 0xf8, 0xb2, 0xc3, 0x97, 0x03, 0x3b, 0x1c, 0x8e, 0xcf, 0x9b, 0xa6, 0xe7, 0xdc, 0x8f, 0xa7, + 0xbc, 0xcf, 0xa7, 0xbc, 0x1f, 0x89, 0x49, 0x87, 0x46, 0x40, 0xac, 0xfb, 0x43, 0x73, 0x44, 0x7d, + 0x62, 0xb2, 0x5f, 0x9d, 0xfd, 0x91, 0x08, 0xfb, 0xd9, 0x11, 0x68, 0x18, 0x8c, 0xcd, 0x30, 0x5a, + 0xaf, 0x11, 0x86, 0x81, 0x7d, 0x3e, 0x0e, 0x89, 0x00, 0xd2, 0xee, 0xc2, 0x7b, 0x7d, 0x83, 0x5e, + 0xb4, 0x3d, 0xf7, 0x85, 0x3d, 0xe8, 0x99, 0x43, 0xe2, 0x18, 0x98, 0xbc, 0x1a, 0x13, 0x1a, 0x6a, + 0x7f, 0x0c, 0xf5, 0xd9, 0x26, 0xea, 0x7b, 0x2e, 0x25, 0xe8, 0x4b, 0x28, 0x30, 0x69, 0xea, 0xca, + 0x3d, 0x65, 0xbb, 0xba, 0xf3, 0x61, 0xf3, 0x4d, 0x8a, 0x13, 0x32, 0x34, 0xe5, 0x2a, 0x9a, 0x3d, + 0x9f, 0x98, 0x98, 0x8f, 0xd4, 0x6e, 0xc3, 0xad, 0xb6, 0xe1, 0x1b, 0xe7, 0xf6, 0xc8, 0x0e, 0x6d, + 0x42, 0xa3, 0x49, 0xc7, 0xb0, 0x95, 0x66, 0xcb, 0x09, 0x7f, 0x06, 0x6b, 0x66, 0x82, 0x2f, 0x27, + 0xde, 0x6d, 0x66, 0xb2, 0x58, 0x73, 0x8f, 0x53, 0x29, 0xe0, 0x14, 0x9c, 0xb6, 0x05, 0xe8, 0xb1, + 0xed, 0x0e, 0x48, 0xe0, 0x07, 0xb6, 0x1b, 0x46, 0xc2, 0xfc, 0x26, 0x0f, 0xb7, 0x52, 0x6c, 0x29, + 0xcc, 0x4b, 0x80, 0x58, 0x8f, 0x4c, 0x94, 0xfc, 0x76, 0x75, 0xe7, 0xeb, 0x8c, 0xa2, 0xcc, 0xc1, + 0x6b, 0xb6, 0x62, 0xb0, 0x8e, 0x1b, 0x06, 0xd7, 0x38, 0x81, 0x8e, 0xbe, 0x81, 0xd2, 0x90, 0x18, + 0xa3, 0x70, 0x58, 0xcf, 0xdd, 0x53, 0xb6, 0x6b, 0x3b, 0x8f, 0x6f, 0x30, 0xcf, 0x01, 0x07, 0xea, + 0x85, 0x46, 0x48, 0xb0, 0x44, 0x45, 0x1f, 0x01, 0x12, 0xff, 0x74, 0x8b, 0x50, 0x33, 0xb0, 0x7d, + 0xe6, 0xc8, 0xf5, 0xfc, 0x3d, 0x65, 0xbb, 0x82, 0x37, 0x45, 0xcb, 0xde, 0xa4, 0xa1, 0xe1, 0xc3, + 0xc6, 0x94, 0xb4, 0x48, 0x85, 0xfc, 0x05, 0xb9, 0xe6, 0x16, 0xa9, 0x60, 0xf6, 0x17, 0xed, 0x43, + 0xf1, 0xd2, 0x18, 0x8d, 0x09, 0x17, 0xb9, 0xba, 0xf3, 0xc9, 0xdb, 0xdc, 0x43, 0xba, 0xe8, 0x44, + 0x0f, 0x58, 0x8c, 0xff, 0x3c, 0xf7, 0x99, 0xa2, 0xed, 0x42, 0x35, 0x21, 0x37, 0xaa, 0x01, 0x9c, + 0x75, 0xf7, 0x3a, 0xfd, 0x4e, 0xbb, 0xdf, 0xd9, 0x53, 0x57, 0xd0, 0x3a, 0x54, 0xce, 0xba, 0x07, + 0x9d, 0xd6, 0x51, 0xff, 0xe0, 0x99, 0xaa, 0xa0, 0x2a, 0xac, 0x46, 0x44, 0x4e, 0xbb, 0x02, 0x84, + 0x89, 0xe9, 0x5d, 0x92, 0x80, 0x39, 0xb2, 0xb4, 0x2a, 0x7a, 0x0f, 0x56, 0x43, 0x83, 0x5e, 0xe8, + 0xb6, 0x25, 0x65, 0x2e, 0x31, 0xf2, 0xd0, 0x42, 0x87, 0x50, 0x1a, 0x1a, 0xae, 0x35, 0x7a, 0xbb, + 0xdc, 0x69, 0x55, 0x33, 0xf0, 0x03, 0x3e, 0x10, 0x4b, 0x00, 0xe6, 0xdd, 0xa9, 0x99, 0x85, 0x01, + 0xb4, 0x67, 0xa0, 0xf6, 0x42, 0x23, 0x08, 0x93, 0xe2, 0x74, 0xa0, 0xc0, 0xe6, 0x97, 0x1e, 0xbd, + 0xc8, 0x9c, 0x62, 0x67, 0x62, 0x3e, 0x5c, 0xfb, 0xff, 0x1c, 0x6c, 0x26, 0xb0, 0xa5, 0xa7, 0x3e, + 0x85, 0x52, 0x40, 0xe8, 0x78, 0x14, 0x72, 0xf8, 0xda, 0xce, 0xa3, 0x8c, 0xf0, 0x33, 0x48, 0x4d, + 0xcc, 0x61, 0xb0, 0x84, 0x43, 0xdb, 0xa0, 0x8a, 0x11, 0x3a, 0x09, 0x02, 0x2f, 0xd0, 0x1d, 0x3a, + 0xe0, 0x5a, 0xab, 0xe0, 0x9a, 0xe0, 0x77, 0x18, 0xfb, 0x98, 0x0e, 0x12, 0x5a, 0xcd, 0xdf, 0x50, + 0xab, 0xc8, 0x00, 0xd5, 0x25, 0xe1, 0x6b, 0x2f, 0xb8, 0xd0, 0x99, 0x6a, 0x03, 0xdb, 0x22, 0xf5, + 0x02, 0x07, 0xfd, 0x34, 0x23, 0x68, 0x57, 0x0c, 0x3f, 0x91, 0xa3, 0xf1, 0x86, 0x9b, 0x66, 0x68, + 0x3f, 0x84, 0x92, 0x58, 0x29, 0xf3, 0xa4, 0xde, 0x59, 0xbb, 0xdd, 0xe9, 0xf5, 0xd4, 0x15, 0x54, + 0x81, 0x22, 0xee, 0xf4, 0x31, 0xf3, 0xb0, 0x0a, 0x14, 0x1f, 0xb7, 0xfa, 0xad, 0x23, 0x35, 0xa7, + 0xfd, 0x00, 0x36, 0x9e, 0x1a, 0x76, 0x98, 0xc5, 0xb9, 0x34, 0x0f, 0xd4, 0x49, 0x5f, 0x69, 0x9d, + 0xc3, 0x94, 0x75, 0xb2, 0xab, 0xa6, 0x73, 0x65, 0x87, 0x53, 0xf6, 0x50, 0x21, 0x4f, 0x82, 0x40, + 0x9a, 0x80, 0xfd, 0xd5, 0x5e, 0xc3, 0x46, 0x2f, 0xf4, 0xfc, 0x4c, 0x9e, 0xff, 0x23, 0x58, 0x65, + 0x31, 0xca, 0x1b, 0x87, 0xd2, 0xf5, 0xef, 0x36, 0x45, 0x0c, 0x6b, 0x46, 0x31, 0xac, 0xb9, 0x27, + 0x63, 0x1c, 0x8e, 0x7a, 0xa2, 0x3b, 0x50, 0xa2, 0xf6, 0xc0, 0x35, 0x46, 0xf2, 0xb4, 0x90, 0x94, + 0x86, 0x98, 0x93, 0x47, 0x13, 0x4b, 0xc7, 0x6f, 0x03, 0xda, 0x23, 0x34, 0x0c, 0xbc, 0xeb, 0x4c, + 0xf2, 0x6c, 0x41, 0xf1, 0x85, 0x17, 0x98, 0x62, 0x23, 0x96, 0xb1, 0x20, 0xd8, 0xa6, 0x4a, 0x81, + 0x48, 0xec, 0x8f, 0x00, 0x1d, 0xba, 0x2c, 0xa6, 0x64, 0x33, 0xc4, 0xdf, 0xe7, 0xe0, 0x56, 0xaa, + 0xbf, 0x34, 0xc6, 0xf2, 0xfb, 0x90, 0x1d, 0x4c, 0x63, 0x2a, 0xf6, 0x21, 0x3a, 0x81, 0x92, 0xe8, + 0x21, 0x35, 0xf9, 0x60, 0x01, 0x20, 0x11, 0xa6, 0x24, 0x9c, 0x84, 0x99, 0xeb, 0xf4, 0xf9, 0x77, + 0xeb, 0xf4, 0xaf, 0x41, 0x8d, 0xd6, 0x41, 0xdf, 0x6a, 0x9b, 0xaf, 0xe1, 0x96, 0xe9, 0x8d, 0x46, + 0xc4, 0x64, 0xde, 0xa0, 0xdb, 0x6e, 0x48, 0x82, 0x4b, 0x63, 0xf4, 0x76, 0xbf, 0x41, 0x93, 0x51, + 0x87, 0x72, 0x90, 0xf6, 0x1c, 0x36, 0x13, 0x13, 0x4b, 0x43, 0x3c, 0x86, 0x22, 0x65, 0x0c, 0x69, + 0x89, 0x8f, 0x17, 0xb4, 0x04, 0xc5, 0x62, 0xb8, 0x76, 0x4b, 0x80, 0x77, 0x2e, 0x89, 0x1b, 0x2f, + 0x4b, 0xdb, 0x83, 0xcd, 0x1e, 0x77, 0xd3, 0x4c, 0x7e, 0x38, 0x71, 0xf1, 0x5c, 0xca, 0xc5, 0xb7, + 0x00, 0x25, 0x51, 0xa4, 0x23, 0x5e, 0xc3, 0x46, 0xe7, 0x8a, 0x98, 0x99, 0x90, 0xeb, 0xb0, 0x6a, + 0x7a, 0x8e, 0x63, 0xb8, 0x56, 0x3d, 0x77, 0x2f, 0xbf, 0x5d, 0xc1, 0x11, 0x99, 0xdc, 0x8b, 0xf9, + 0xac, 0x7b, 0x51, 0xfb, 0x5b, 0x05, 0xd4, 0xc9, 0xdc, 0x52, 0x91, 0x4c, 0xfa, 0xd0, 0x62, 0x40, + 0x6c, 0xee, 0x35, 0x2c, 0x29, 0xc9, 0x8f, 0x8e, 0x0b, 0xc1, 0x27, 0x41, 0x90, 0x38, 0x8e, 0xf2, + 0x37, 0x3c, 0x8e, 0xb4, 0x03, 0xf8, 0x9d, 0x48, 0x9c, 0x5e, 0x18, 0x10, 0xc3, 0xb1, 0xdd, 0xc1, + 0xe1, 0xc9, 0x89, 0x4f, 0x84, 0xe0, 0x08, 0x41, 0xc1, 0x32, 0x42, 0x43, 0x0a, 0xc6, 0xff, 0xb3, + 0x4d, 0x6f, 0x8e, 0x3c, 0x1a, 0x6f, 0x7a, 0x4e, 0x68, 0xff, 0x9e, 0x87, 0xfa, 0x0c, 0x54, 0xa4, + 0xde, 0xe7, 0x50, 0xa4, 0x24, 0x1c, 0xfb, 0xd2, 0x55, 0x3a, 0x99, 0x05, 0x9e, 0x8f, 0xd7, 0xec, + 0x31, 0x30, 0x2c, 0x30, 0xd1, 0x00, 0xca, 0x61, 0x78, 0xad, 0x53, 0xfb, 0xe7, 0x51, 0x42, 0x70, + 0x74, 0x53, 0xfc, 0x3e, 0x09, 0x1c, 0xdb, 0x35, 0x46, 0x3d, 0xfb, 0xe7, 0x04, 0xaf, 0x86, 0xe1, + 0x35, 0xfb, 0x83, 0x9e, 0x31, 0x87, 0xb7, 0x6c, 0x57, 0xaa, 0xbd, 0xbd, 0xec, 0x2c, 0x09, 0x05, + 0x63, 0x81, 0xd8, 0x38, 0x82, 0x22, 0x5f, 0xd3, 0x32, 0x8e, 0xa8, 0x42, 0x3e, 0x0c, 0xaf, 0xb9, + 0x50, 0x65, 0xcc, 0xfe, 0x36, 0x1e, 0xc2, 0x5a, 0x72, 0x05, 0xcc, 0x91, 0x86, 0xc4, 0x1e, 0x0c, + 0x85, 0x83, 0x15, 0xb1, 0xa4, 0x98, 0x25, 0x5f, 0xdb, 0x96, 0x4c, 0x59, 0x8b, 0x58, 0x10, 0xda, + 0xbf, 0xe4, 0xe0, 0xee, 0x1c, 0xcd, 0x48, 0x67, 0x7d, 0x9e, 0x72, 0xd6, 0x77, 0xa4, 0x85, 0xc8, + 0xe3, 0x9f, 0xa7, 0x3c, 0xfe, 0x1d, 0x82, 0xb3, 0x6d, 0x73, 0x07, 0x4a, 0xe4, 0xca, 0x0e, 0x89, + 0x25, 0x55, 0x25, 0xa9, 0xc4, 0x76, 0x2a, 0xdc, 0x74, 0x3b, 0x7d, 0x02, 0x5b, 0xed, 0x80, 0x18, + 0x21, 0x91, 0x47, 0x79, 0xe4, 0xff, 0x77, 0xa1, 0x6c, 0x8c, 0x46, 0x9e, 0x39, 0x31, 0xeb, 0x2a, + 0xa7, 0x0f, 0x2d, 0xed, 0x5b, 0x05, 0x6e, 0x4f, 0x8d, 0x91, 0x9a, 0x3e, 0x87, 0x9a, 0x4d, 0xbd, + 0x11, 0x5f, 0x84, 0x9e, 0xb8, 0xc5, 0xfd, 0x78, 0xb1, 0x70, 0x72, 0x18, 0x61, 0xf0, 0x4b, 0xdd, + 0xba, 0x9d, 0x24, 0xb9, 0x57, 0xf1, 0xc9, 0x2d, 0xb9, 0x9b, 0x23, 0x52, 0xfb, 0x07, 0x05, 0x6e, + 0xcb, 0x28, 0x9e, 0x79, 0x31, 0x73, 0x44, 0xce, 0xbd, 0x6b, 0x91, 0xb5, 0x3a, 0xdc, 0x99, 0x96, + 0x4b, 0x9e, 0xeb, 0xff, 0x51, 0x00, 0x34, 0x7b, 0x83, 0x44, 0xdf, 0x83, 0x35, 0x4a, 0x5c, 0x4b, + 0x17, 0x31, 0x41, 0x84, 0xab, 0x32, 0xae, 0x32, 0x9e, 0x08, 0x0e, 0x94, 0x1d, 0x73, 0xe4, 0x4a, + 0x4a, 0x5b, 0xc6, 0xfc, 0x3f, 0x1a, 0xc2, 0xda, 0x0b, 0xaa, 0xc7, 0x73, 0x73, 0xa7, 0xa9, 0x65, + 0x3e, 0xba, 0x66, 0xe5, 0x68, 0x3e, 0xee, 0xc5, 0xeb, 0xc2, 0xd5, 0x17, 0x34, 0x26, 0xd0, 0x2f, + 0x15, 0x78, 0x2f, 0x4a, 0x1d, 0x26, 0xea, 0x73, 0x3c, 0x8b, 0xd0, 0x7a, 0xe1, 0x5e, 0x7e, 0xbb, + 0xb6, 0x73, 0x7a, 0x03, 0xfd, 0xcd, 0x30, 0x8f, 0x3d, 0x8b, 0xe0, 0xdb, 0xee, 0x1c, 0x2e, 0x45, + 0x4d, 0xb8, 0xe5, 0x8c, 0x69, 0xa8, 0x0b, 0x2f, 0xd0, 0x65, 0xa7, 0x7a, 0x91, 0xeb, 0x65, 0x93, + 0x35, 0xa5, 0x7c, 0x15, 0x5d, 0xc0, 0xba, 0xe3, 0x8d, 0xdd, 0x50, 0x37, 0xf9, 0x1d, 0x87, 0xd6, + 0x4b, 0x0b, 0x5d, 0x7e, 0xe7, 0x68, 0xe9, 0x98, 0xc1, 0x89, 0x1b, 0x13, 0xc5, 0x6b, 0x4e, 0x82, + 0xd2, 0x9a, 0x50, 0x4d, 0xe8, 0x10, 0x95, 0xa1, 0xd0, 0x3d, 0xe9, 0x76, 0xd4, 0x15, 0x04, 0x50, + 0x6a, 0x1f, 0xe0, 0x93, 0x93, 0xbe, 0x48, 0xfb, 0x0f, 0x8f, 0x5b, 0xfb, 0x1d, 0x35, 0xa7, 0x75, + 0x60, 0x2d, 0x89, 0x86, 0x10, 0xd4, 0xce, 0xba, 0x4f, 0xba, 0x27, 0x4f, 0xbb, 0xfa, 0xf1, 0xc9, + 0x59, 0xb7, 0xcf, 0x2e, 0x0c, 0x35, 0x80, 0x56, 0xf7, 0xd9, 0x84, 0x5e, 0x87, 0x4a, 0xf7, 0x24, + 0x22, 0x95, 0x46, 0x4e, 0x55, 0xb4, 0xff, 0xcd, 0xc1, 0xd6, 0x3c, 0xc5, 0x22, 0x0b, 0x0a, 0xcc, + 0x48, 0xf2, 0xca, 0xf6, 0xee, 0x6d, 0xc4, 0xd1, 0x99, 0x6f, 0xfa, 0x86, 0x3c, 0xa3, 0x2b, 0x98, + 0xff, 0x47, 0x3a, 0x94, 0x46, 0xc6, 0x39, 0x19, 0xd1, 0x7a, 0x9e, 0x3f, 0x6a, 0xec, 0xdf, 0x64, + 0xee, 0x23, 0x8e, 0x24, 0x5e, 0x34, 0x24, 0x6c, 0x63, 0x17, 0xaa, 0x09, 0xf6, 0x9c, 0xa7, 0x83, + 0xad, 0xe4, 0xd3, 0x41, 0x25, 0xf9, 0x0e, 0xf0, 0x68, 0x56, 0x5b, 0x6c, 0x35, 0xcc, 0x5c, 0x07, + 0x27, 0xbd, 0xbe, 0xb8, 0xa4, 0xed, 0xe3, 0x93, 0xb3, 0x53, 0x55, 0x61, 0xcc, 0x7e, 0xab, 0xf7, + 0x44, 0xcd, 0xc5, 0xd6, 0xcc, 0x6b, 0xcf, 0xa1, 0xb2, 0xd7, 0xed, 0x09, 0xa3, 0xb1, 0x03, 0x8a, + 0x92, 0x80, 0x2d, 0x81, 0xbf, 0xdf, 0x54, 0x70, 0x44, 0xa2, 0x06, 0x94, 0x29, 0x31, 0x02, 0x73, + 0x48, 0xa8, 0x8c, 0x88, 0x31, 0xcd, 0x46, 0x79, 0xfc, 0x1d, 0x44, 0x28, 0xa8, 0x82, 0x23, 0x52, + 0xfb, 0xbf, 0x55, 0x80, 0xc9, 0x9d, 0x1c, 0xd5, 0x20, 0x17, 0x9f, 0x62, 0x39, 0xdb, 0x62, 0xca, + 0x76, 0x0d, 0x27, 0x5a, 0x15, 0xff, 0x8f, 0x76, 0xe0, 0xb6, 0x43, 0x07, 0xbe, 0x61, 0x5e, 0xe8, + 0xf2, 0x2a, 0x2d, 0x9c, 0x9d, 0x9f, 0x08, 0x6b, 0xf8, 0x96, 0x6c, 0x94, 0xbe, 0x2c, 0x70, 0x8f, + 0x20, 0x4f, 0xdc, 0x4b, 0xbe, 0x7b, 0xab, 0x3b, 0x9f, 0x2f, 0xfc, 0x56, 0xd0, 0xec, 0xb8, 0x97, + 0xc2, 0x20, 0x0c, 0x06, 0xe9, 0x00, 0x16, 0xb9, 0xb4, 0x4d, 0xa2, 0x33, 0xd0, 0x22, 0x07, 0xfd, + 0x72, 0x71, 0xd0, 0x3d, 0x8e, 0x11, 0x43, 0x57, 0xac, 0x88, 0x46, 0x5d, 0xa8, 0x04, 0x84, 0x7a, + 0xe3, 0xc0, 0x24, 0x62, 0x0b, 0x67, 0x4f, 0xe7, 0x71, 0x34, 0x0e, 0x4f, 0x20, 0xd0, 0x1e, 0x94, + 0xf8, 0xce, 0xa5, 0xf5, 0x55, 0x2e, 0xec, 0x87, 0x19, 0xc1, 0xf8, 0x76, 0xc5, 0x72, 0x2c, 0xda, + 0x87, 0x55, 0x21, 0x22, 0xad, 0x97, 0x39, 0xcc, 0x47, 0x59, 0x8f, 0x15, 0x3e, 0x0a, 0x47, 0xa3, + 0x99, 0x55, 0xc7, 0x94, 0x04, 0xf5, 0x8a, 0xb0, 0x2a, 0xfb, 0x8f, 0xde, 0x87, 0x8a, 0x88, 0x62, + 0x96, 0x1d, 0xd4, 0x81, 0x37, 0x88, 0xb0, 0xb6, 0x67, 0x07, 0xe8, 0x03, 0xa8, 0x8a, 0x8c, 0x44, + 0xe7, 0x5b, 0xaf, 0xca, 0x9b, 0x41, 0xb0, 0x4e, 0xd9, 0x06, 0x14, 0x1d, 0x48, 0x10, 0x88, 0x0e, + 0x6b, 0x71, 0x07, 0x12, 0x04, 0xbc, 0xc3, 0xef, 0xc3, 0x06, 0xcf, 0xe3, 0x06, 0x81, 0x37, 0xf6, + 0x75, 0xee, 0x53, 0xeb, 0xbc, 0xd3, 0x3a, 0x63, 0xef, 0x33, 0x6e, 0x97, 0x39, 0xd7, 0x5d, 0x28, + 0xbf, 0xf4, 0xce, 0x45, 0x87, 0x9a, 0x08, 0xa6, 0x2f, 0xbd, 0xf3, 0xa8, 0x29, 0x8e, 0xb3, 0x1b, + 0xe9, 0x38, 0xfb, 0x0a, 0xee, 0xcc, 0x06, 0x0c, 0x1e, 0x6f, 0xd5, 0x9b, 0xc7, 0xdb, 0x2d, 0x77, + 0xde, 0x61, 0xf7, 0x15, 0xe4, 0x2d, 0x97, 0xd6, 0x37, 0x17, 0x72, 0x8e, 0x78, 0x1f, 0x63, 0x36, + 0xb8, 0xf1, 0x29, 0x94, 0x23, 0xef, 0x5b, 0xe4, 0x48, 0x69, 0x3c, 0x84, 0x5a, 0xda, 0x77, 0x17, + 0x3a, 0x90, 0xfe, 0x53, 0x81, 0x4a, 0xec, 0xa5, 0xc8, 0x85, 0x5b, 0x5c, 0x8b, 0x2c, 0xc9, 0xd1, + 0x27, 0x4e, 0x2f, 0x52, 0xab, 0x2f, 0x32, 0xae, 0xab, 0x15, 0x21, 0xc8, 0x7b, 0x9c, 0xdc, 0x01, + 0x28, 0x46, 0x9e, 0xcc, 0xf7, 0x0d, 0x6c, 0x8c, 0x6c, 0x77, 0x7c, 0x95, 0x98, 0x4b, 0xe4, 0x44, + 0x7f, 0x90, 0x71, 0xae, 0x23, 0x36, 0x7a, 0x32, 0x47, 0x6d, 0x94, 0xa2, 0xb5, 0x6f, 0x73, 0x70, + 0x67, 0xbe, 0x38, 0xa8, 0x0b, 0x79, 0xd3, 0x1f, 0xcb, 0xa5, 0x3d, 0x5c, 0x74, 0x69, 0x6d, 0x7f, + 0x3c, 0x99, 0x95, 0x01, 0xa1, 0xa7, 0x50, 0x72, 0x88, 0xe3, 0x05, 0xd7, 0x72, 0x05, 0x8f, 0x16, + 0x85, 0x3c, 0xe6, 0xa3, 0x27, 0xa8, 0x12, 0x0e, 0x61, 0x28, 0x4b, 0x9f, 0xa3, 0xf2, 0x74, 0x5b, + 0xf0, 0xc9, 0x24, 0x82, 0xc4, 0x31, 0x8e, 0xf6, 0x29, 0xdc, 0x9e, 0xbb, 0x14, 0xf4, 0xbb, 0x00, + 0xa6, 0x3f, 0xd6, 0xf9, 0x93, 0xb6, 0xb0, 0x7b, 0x1e, 0x57, 0x4c, 0x7f, 0xdc, 0xe3, 0x0c, 0xed, + 0x01, 0xd4, 0xdf, 0x24, 0x2f, 0x3b, 0x33, 0x84, 0xc4, 0xba, 0x73, 0xce, 0x75, 0x90, 0xc7, 0x65, + 0xc1, 0x38, 0x3e, 0xd7, 0x7e, 0x95, 0x83, 0x8d, 0x29, 0x71, 0xd8, 0x95, 0x43, 0x9c, 0x41, 0xd1, + 0x65, 0x4e, 0x50, 0xec, 0x40, 0x32, 0x6d, 0x2b, 0x7a, 0x06, 0xe4, 0xff, 0x79, 0x28, 0xf2, 0xe5, + 0x13, 0x5d, 0xce, 0xf6, 0x99, 0x43, 0x3b, 0xe7, 0x76, 0x48, 0xf9, 0xad, 0xa4, 0x88, 0x05, 0x81, + 0x9e, 0x41, 0x2d, 0x20, 0x3c, 0x04, 0x5a, 0xba, 0xef, 0x05, 0x61, 0xa4, 0xb0, 0x9d, 0xc5, 0x14, + 0x76, 0xea, 0x05, 0x21, 0x5e, 0x8f, 0x90, 0x18, 0x45, 0xd1, 0x53, 0x58, 0xb7, 0xae, 0x5d, 0xc3, + 0xb1, 0x4d, 0x89, 0x5c, 0x5a, 0x1a, 0x79, 0x4d, 0x02, 0x71, 0x60, 0x6d, 0x17, 0xaa, 0x89, 0x46, + 0xb6, 0x30, 0x9e, 0x65, 0x48, 0x9d, 0x08, 0x22, 0xbd, 0x7f, 0x8b, 0x72, 0xff, 0x6a, 0xff, 0x94, + 0x83, 0x5a, 0x7a, 0x03, 0x44, 0xf6, 0xf3, 0x49, 0x60, 0x7b, 0x56, 0xc2, 0x7e, 0xa7, 0x9c, 0xc1, + 0x6c, 0xc4, 0x9a, 0x5f, 0x8d, 0xbd, 0xd0, 0x88, 0x6c, 0x64, 0xfa, 0xe3, 0x3f, 0x64, 0xf4, 0x94, + 0xed, 0xf3, 0x53, 0xb6, 0x47, 0x1f, 0x02, 0x92, 0xf6, 0x1d, 0xd9, 0x8e, 0x1d, 0xea, 0xe7, 0xd7, + 0x21, 0x11, 0xfa, 0xcf, 0x63, 0x55, 0xb4, 0x1c, 0xb1, 0x86, 0xaf, 0x18, 0x1f, 0x69, 0xb0, 0xee, + 0x79, 0x8e, 0x4e, 0x4d, 0x2f, 0x20, 0xba, 0x61, 0xbd, 0xe4, 0x59, 0x72, 0x1e, 0x57, 0x3d, 0xcf, + 0xe9, 0x31, 0x5e, 0xcb, 0x7a, 0xc9, 0xe2, 0x84, 0xe9, 0x8f, 0x29, 0x09, 0x75, 0xf6, 0xc3, 0x43, + 0x6b, 0x05, 0x83, 0x60, 0xb5, 0xfd, 0x31, 0x4d, 0x74, 0x70, 0x88, 0xc3, 0xc2, 0x65, 0xa2, 0xc3, + 0x31, 0x71, 0xd8, 0x2c, 0x6b, 0xa7, 0x24, 0x30, 0x89, 0x1b, 0xf6, 0x6d, 0xf3, 0x82, 0x45, 0x42, + 0x65, 0x5b, 0xc1, 0x29, 0x9e, 0xf6, 0x33, 0x28, 0xf2, 0xc8, 0xc9, 0x16, 0xcf, 0xa3, 0x0e, 0x0f, + 0x4a, 0x42, 0xbd, 0x65, 0xc6, 0xe0, 0x21, 0xe9, 0x7d, 0xa8, 0x0c, 0x3d, 0x2a, 0x43, 0x9a, 0xf0, + 0xbc, 0x32, 0x63, 0xf0, 0xc6, 0x06, 0x94, 0x03, 0x62, 0x58, 0x9e, 0x3b, 0x8a, 0x5e, 0x12, 0x62, + 0x5a, 0x7b, 0x05, 0x25, 0x71, 0xfc, 0xde, 0x00, 0xff, 0x23, 0x40, 0xa6, 0x88, 0x85, 0x3e, 0x09, + 0x1c, 0x9b, 0x52, 0x99, 0x9c, 0xf1, 0xcf, 0x57, 0xa2, 0xe5, 0x74, 0xd2, 0xa0, 0xfd, 0x97, 0x22, + 0xd2, 0x34, 0xf1, 0x61, 0x81, 0xe5, 0x73, 0xcc, 0xd3, 0xd8, 0x35, 0x4c, 0xbc, 0x60, 0x44, 0x24, + 0xbb, 0xbc, 0xcb, 0x6c, 0x2c, 0xb7, 0xec, 0x77, 0x19, 0x09, 0x10, 0xbd, 0x67, 0x12, 0x79, 0xd3, + 0x5b, 0xf4, 0x3d, 0x93, 0x88, 0xf7, 0x4c, 0xc2, 0xee, 0x9b, 0x32, 0x4f, 0x14, 0x70, 0x05, 0x9e, + 0x26, 0x56, 0xad, 0xf8, 0xd1, 0x98, 0x68, 0xff, 0xa3, 0xc4, 0x67, 0x45, 0xf4, 0xb8, 0x8b, 0xbe, + 0x81, 0x32, 0xdb, 0x76, 0xba, 0x63, 0xf8, 0xf2, 0x53, 0x65, 0x7b, 0xb9, 0x77, 0xe3, 0x26, 0xdb, + 0x65, 0xc7, 0x86, 0x2f, 0xb2, 0xbc, 0x55, 0x5f, 0x50, 0xec, 0xcc, 0x31, 0xac, 0xc9, 0x99, 0xc3, + 0xfe, 0xa3, 0xef, 0x43, 0xcd, 0x18, 0x87, 0x9e, 0x6e, 0x58, 0x97, 0x24, 0x08, 0x6d, 0x4a, 0xa4, + 0xed, 0xd7, 0x19, 0xb7, 0x15, 0x31, 0x1b, 0x9f, 0xc3, 0x5a, 0x12, 0xf3, 0x6d, 0xd1, 0xb7, 0x98, + 0x8c, 0xbe, 0x7f, 0x02, 0x30, 0x79, 0x28, 0x61, 0x3e, 0x42, 0xae, 0x6c, 0x76, 0x5d, 0x94, 0xf7, + 0xa6, 0x22, 0x2e, 0x33, 0x46, 0x9b, 0xdd, 0x10, 0xd2, 0xaf, 0xb8, 0xc5, 0xe8, 0x15, 0x97, 0xed, + 0x5a, 0xb6, 0xd1, 0x2e, 0xec, 0xd1, 0x28, 0x7e, 0xbc, 0xa9, 0x78, 0x9e, 0xf3, 0x84, 0x33, 0xb4, + 0xdf, 0xe4, 0x84, 0xaf, 0x88, 0xf7, 0xf8, 0x4c, 0x29, 0xfd, 0xbb, 0x32, 0xf5, 0x2e, 0x00, 0x0d, + 0x8d, 0x80, 0xa5, 0x12, 0x46, 0xf4, 0x7c, 0xd4, 0x98, 0x79, 0x06, 0xee, 0x47, 0x65, 0x05, 0xb8, + 0x22, 0x7b, 0xb7, 0x42, 0xf4, 0x05, 0xac, 0x99, 0x9e, 0xe3, 0x8f, 0x88, 0x1c, 0x5c, 0x7c, 0xeb, + 0xe0, 0x6a, 0xdc, 0xbf, 0x15, 0x26, 0x1e, 0xad, 0x4a, 0x37, 0x7d, 0xb4, 0xfa, 0x57, 0x45, 0x7c, + 0x56, 0x48, 0x7e, 0xd5, 0x40, 0x83, 0x39, 0x9f, 0xce, 0xf7, 0x97, 0xfc, 0x44, 0xf2, 0x5d, 0xdf, + 0xcd, 0x1b, 0x5f, 0x64, 0xf9, 0x50, 0xfd, 0xe6, 0xe4, 0xee, 0xdf, 0xf2, 0x50, 0x89, 0xbf, 0x28, + 0xcc, 0xd8, 0xfe, 0x33, 0xa8, 0xc4, 0x35, 0x1d, 0xf2, 0x80, 0xf8, 0x4e, 0xf3, 0xc4, 0x9d, 0xd1, + 0x0b, 0x40, 0xc6, 0x60, 0x10, 0x27, 0x6d, 0xfa, 0x98, 0x1a, 0x83, 0xe8, 0x7b, 0xce, 0x67, 0x0b, + 0xe8, 0x21, 0x8a, 0x5b, 0x67, 0x6c, 0x3c, 0x56, 0x8d, 0xc1, 0x20, 0xc5, 0x41, 0x7f, 0x0a, 0xb7, + 0xd3, 0x73, 0xe8, 0xe7, 0xd7, 0xba, 0x6f, 0x5b, 0xf2, 0xea, 0x78, 0xb0, 0xe8, 0x47, 0x95, 0x66, + 0x0a, 0xfe, 0xab, 0xeb, 0x53, 0xdb, 0x12, 0x3a, 0x47, 0xc1, 0x4c, 0x43, 0xe3, 0xcf, 0xe1, 0xbd, + 0x37, 0x74, 0x9f, 0x63, 0x83, 0x6e, 0xba, 0x58, 0x60, 0x79, 0x25, 0x24, 0xac, 0xf7, 0x6b, 0x45, + 0x7c, 0xfb, 0x49, 0xeb, 0xa4, 0x95, 0xcc, 0x5b, 0xef, 0x67, 0x9c, 0xa7, 0x7d, 0x7a, 0x26, 0xe0, + 0x79, 0xaa, 0xfa, 0xf5, 0x54, 0xaa, 0x9a, 0x35, 0x89, 0x11, 0x19, 0x9f, 0x00, 0x92, 0x08, 0xda, + 0x3f, 0xe7, 0xa1, 0x1c, 0xa1, 0xf3, 0x8b, 0xdf, 0x35, 0x0d, 0x89, 0xa3, 0xc7, 0x4f, 0x3f, 0x0a, + 0x06, 0xc1, 0xe2, 0xcf, 0x1c, 0xef, 0x43, 0x85, 0xdd, 0x2f, 0x45, 0x73, 0x8e, 0x37, 0x97, 0x19, + 0x83, 0x37, 0x7e, 0x00, 0xd5, 0xd0, 0x0b, 0x8d, 0x91, 0x1e, 0xf2, 0x58, 0x9e, 0x17, 0xa3, 0x39, + 0x8b, 0x47, 0x72, 0xf4, 0x43, 0xd8, 0x0c, 0x87, 0x81, 0x17, 0x86, 0x23, 0x96, 0xdf, 0xf1, 0x8c, + 0x46, 0x24, 0x20, 0x05, 0xac, 0xc6, 0x0d, 0x22, 0xd3, 0xa1, 0xec, 0xf4, 0x9e, 0x74, 0x66, 0xae, + 0xcb, 0x0f, 0x91, 0x02, 0x5e, 0x8f, 0xb9, 0xcc, 0xb5, 0x59, 0xf0, 0xf4, 0x45, 0xb6, 0xc0, 0xcf, + 0x0a, 0x05, 0x47, 0x24, 0xd2, 0x61, 0xc3, 0x21, 0x06, 0x1d, 0x07, 0xc4, 0xd2, 0x5f, 0xd8, 0x64, + 0x64, 0x89, 0xfb, 0x7a, 0x2d, 0x73, 0xfa, 0x1d, 0xa9, 0xa5, 0xf9, 0x98, 0x8f, 0xc6, 0xb5, 0x08, + 0x4e, 0xd0, 0x2c, 0x73, 0x10, 0xff, 0xd0, 0x06, 0x54, 0x7b, 0xcf, 0x7a, 0xfd, 0xce, 0xb1, 0x7e, + 0x7c, 0xb2, 0xd7, 0x91, 0xf5, 0x20, 0xbd, 0x0e, 0x16, 0xa4, 0xc2, 0xda, 0xfb, 0x27, 0xfd, 0xd6, + 0x91, 0xde, 0x3f, 0x6c, 0x3f, 0xe9, 0xa9, 0x39, 0x74, 0x1b, 0x36, 0xfb, 0x07, 0xf8, 0xa4, 0xdf, + 0x3f, 0xea, 0xec, 0xe9, 0xa7, 0x1d, 0x7c, 0x78, 0xb2, 0xd7, 0x53, 0xf3, 0x08, 0x41, 0x6d, 0xc2, + 0xee, 0x1f, 0x1e, 0x77, 0xd4, 0x02, 0xaa, 0xc2, 0xea, 0x69, 0x07, 0xb7, 0x3b, 0xdd, 0xbe, 0x5a, + 0xd4, 0x7e, 0x95, 0x87, 0x6a, 0xc2, 0x8a, 0xcc, 0x91, 0x03, 0x2a, 0xf2, 0xfc, 0x02, 0x66, 0x7f, + 0xf9, 0xf7, 0x2b, 0xc3, 0x1c, 0x0a, 0xeb, 0x14, 0xb0, 0x20, 0x78, 0x6e, 0x6f, 0x5c, 0x25, 0xf6, + 0x79, 0x01, 0x97, 0x1d, 0xe3, 0x4a, 0x80, 0x7c, 0x0f, 0xd6, 0x2e, 0x48, 0xe0, 0x92, 0x91, 0x6c, + 0x17, 0x16, 0xa9, 0x0a, 0x9e, 0xe8, 0xb2, 0x0d, 0xaa, 0xec, 0x32, 0x81, 0x11, 0xe6, 0xa8, 0x09, + 0xfe, 0x71, 0x04, 0xb6, 0x05, 0x45, 0xd1, 0xbc, 0x2a, 0xe6, 0xe7, 0x04, 0x0b, 0x53, 0xf4, 0xb5, + 0xe1, 0xf3, 0xfc, 0xae, 0x80, 0xf9, 0x7f, 0x74, 0x3e, 0x6b, 0x9f, 0x12, 0xb7, 0xcf, 0xee, 0xe2, + 0xee, 0xfc, 0x26, 0x13, 0x0d, 0x63, 0x13, 0xad, 0x42, 0x1e, 0x47, 0x45, 0x14, 0xed, 0x56, 0xfb, + 0x80, 0x99, 0x65, 0x1d, 0x2a, 0xc7, 0xad, 0x9f, 0xea, 0x67, 0x3d, 0xfe, 0xa2, 0x8a, 0x54, 0x58, + 0x7b, 0xd2, 0xc1, 0xdd, 0xce, 0x91, 0xe4, 0xe4, 0xd1, 0x16, 0xa8, 0x92, 0x33, 0xe9, 0x57, 0x60, + 0x08, 0xe2, 0x6f, 0x11, 0x95, 0xa1, 0xd0, 0x7b, 0xda, 0x3a, 0x55, 0x4b, 0xda, 0x7f, 0xe7, 0x60, + 0x43, 0x84, 0x85, 0xf8, 0x73, 0xef, 0x9b, 0x3f, 0x77, 0x25, 0x1f, 0x3f, 0x72, 0xe9, 0xc7, 0x8f, + 0x28, 0x09, 0xe5, 0x51, 0x3d, 0x3f, 0x49, 0x42, 0xf9, 0xa3, 0x49, 0xea, 0xc4, 0x2f, 0x2c, 0x72, + 0xe2, 0xd7, 0x61, 0xd5, 0x21, 0x34, 0xb6, 0x5b, 0x05, 0x47, 0x24, 0xb2, 0xa1, 0x6a, 0xb8, 0xae, + 0x17, 0x1a, 0xe2, 0x45, 0xb1, 0xb4, 0x50, 0x30, 0x9c, 0x5a, 0x71, 0xb3, 0x35, 0x41, 0x12, 0x07, + 0x73, 0x12, 0xbb, 0xf1, 0x13, 0x50, 0xa7, 0x3b, 0x2c, 0x12, 0x0e, 0x7f, 0xf0, 0xc9, 0x24, 0x1a, + 0x12, 0xb6, 0x2f, 0xe4, 0x7b, 0xb7, 0xba, 0xc2, 0x08, 0x7c, 0xd6, 0xed, 0x1e, 0x76, 0xf7, 0x55, + 0x05, 0x01, 0x94, 0x3a, 0x3f, 0x3d, 0xec, 0x77, 0xf6, 0xd4, 0xdc, 0xce, 0xaf, 0x37, 0xa1, 0x24, + 0x84, 0x44, 0xdf, 0xca, 0x4c, 0x20, 0x59, 0x4a, 0x88, 0x7e, 0xb2, 0x70, 0x46, 0x9d, 0x2a, 0x4f, + 0x6c, 0x3c, 0x5a, 0x7a, 0xbc, 0xfc, 0xac, 0xb3, 0x82, 0xfe, 0x5a, 0x81, 0xb5, 0xd4, 0x27, 0x9d, + 0xac, 0x2f, 0xaa, 0x73, 0x2a, 0x17, 0x1b, 0x3f, 0x5e, 0x6a, 0x6c, 0x2c, 0xcb, 0x2f, 0x15, 0xa8, + 0x26, 0x6a, 0xf6, 0xd0, 0xee, 0x32, 0x75, 0x7e, 0x42, 0x92, 0xcf, 0x97, 0x2f, 0x11, 0xd4, 0x56, + 0x3e, 0x56, 0xd0, 0x5f, 0x29, 0x50, 0x4d, 0x54, 0xaf, 0x65, 0x16, 0x65, 0xb6, 0xd6, 0x2e, 0xb3, + 0x28, 0xf3, 0x8a, 0xe5, 0x56, 0xd0, 0x5f, 0x28, 0x50, 0x89, 0x2b, 0xd1, 0xd0, 0x83, 0xc5, 0x6b, + 0xd7, 0x84, 0x10, 0x9f, 0x2d, 0x5b, 0xf4, 0xa6, 0xad, 0xa0, 0x3f, 0x83, 0x72, 0x54, 0xb6, 0x85, + 0xb2, 0x46, 0xaf, 0xa9, 0x9a, 0xb0, 0xc6, 0x83, 0x85, 0xc7, 0x25, 0xa7, 0x8f, 0x6a, 0xa9, 0x32, + 0x4f, 0x3f, 0x55, 0xf5, 0xd5, 0x78, 0xb0, 0xf0, 0xb8, 0x78, 0x7a, 0xe6, 0x09, 0x89, 0x92, 0xab, + 0xcc, 0x9e, 0x30, 0x5b, 0xeb, 0x95, 0xd9, 0x13, 0xe6, 0x55, 0x78, 0x09, 0x41, 0x12, 0x45, 0x5b, + 0x99, 0x05, 0x99, 0x2d, 0x0c, 0xcb, 0x2c, 0xc8, 0x9c, 0x1a, 0x31, 0x6d, 0x05, 0xfd, 0x42, 0x49, + 0xde, 0x0b, 0x1e, 0x2c, 0x5c, 0x9b, 0xb4, 0xa0, 0x4b, 0xce, 0x54, 0x47, 0xf1, 0x0d, 0xfa, 0x0b, + 0xf9, 0x8a, 0x21, 0x4a, 0x9b, 0xd0, 0x22, 0x60, 0xa9, 0x6a, 0xa8, 0xc6, 0xa7, 0xcb, 0x05, 0x1b, + 0x2e, 0xc4, 0x5f, 0x2a, 0x00, 0x93, 0x22, 0xa8, 0xcc, 0x42, 0xcc, 0x54, 0x5f, 0x35, 0x76, 0x97, + 0x18, 0x99, 0xdc, 0x20, 0x51, 0x91, 0x46, 0xe6, 0x0d, 0x32, 0x55, 0xa4, 0x95, 0x79, 0x83, 0x4c, + 0x17, 0x58, 0x69, 0x2b, 0xe8, 0x1f, 0x15, 0xd8, 0x9c, 0x29, 0x12, 0x41, 0x8f, 0x6e, 0x58, 0x27, + 0xd4, 0xf8, 0x72, 0x79, 0x80, 0x48, 0xb4, 0x6d, 0xe5, 0x63, 0x05, 0xfd, 0x8d, 0x02, 0xeb, 0xe9, + 0x0f, 0xeb, 0x99, 0xa3, 0xd4, 0x9c, 0x72, 0x93, 0xc6, 0xc3, 0xe5, 0x06, 0xc7, 0xda, 0xfa, 0x3b, + 0x05, 0x6a, 0xe9, 0x1a, 0x0b, 0xf4, 0x70, 0xb1, 0x63, 0x61, 0x4a, 0xa0, 0x2f, 0x96, 0x1c, 0x1d, + 0x49, 0xf4, 0xd5, 0xea, 0x1f, 0x15, 0x45, 0xf6, 0x56, 0xe2, 0x3f, 0x3f, 0xfa, 0x6d, 0x00, 0x00, + 0x00, 0xff, 0xff, 0x29, 0x1e, 0x98, 0x4c, 0x27, 0x32, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/plugins/drivers/proto/driver.proto b/plugins/drivers/proto/driver.proto index 89c9563d83d6..5284feef5ae8 100644 --- a/plugins/drivers/proto/driver.proto +++ b/plugins/drivers/proto/driver.proto @@ -394,6 +394,12 @@ message NetworkIsolationSpec { map labels = 3; } +message DNSConfig { + repeated string servers = 1; + repeated string searches = 2; + repeated string options = 3; +} + message TaskConfig { // Id of the task, recommended to the globally unique, must be unique to the driver. @@ -449,6 +455,9 @@ message TaskConfig { // NetworkIsolationSpec specifies the configuration for the network namespace // to use for the task. *Only supported on Linux NetworkIsolationSpec network_isolation_spec = 16; + + // DNSConfig is the configuration for task DNS resolvers and other options + DNSConfig dns = 17; } message Resources { diff --git a/plugins/drivers/testutils/dns_testing.go b/plugins/drivers/testutils/dns_testing.go new file mode 100644 index 000000000000..f27a8fead6c3 --- /dev/null +++ b/plugins/drivers/testutils/dns_testing.go @@ -0,0 +1,49 @@ +package testutils + +import ( + "strings" + "testing" + + dresolvconf "github.com/docker/libnetwork/resolvconf" + dtypes "github.com/docker/libnetwork/types" + "github.com/hashicorp/nomad/plugins/drivers" + "github.com/stretchr/testify/require" +) + +// TestTaskDNSConfig asserts that a task is running with the given DNSConfig +func TestTaskDNSConfig(t *testing.T, driver *DriverHarness, taskID string, dns *drivers.DNSConfig) { + t.Run("dns_config", func(t *testing.T) { + caps, err := driver.Capabilities() + require.NoError(t, err) + + isolated := (caps.FSIsolation != drivers.FSIsolationNone) + if !isolated { + t.Skip("dns config not supported on non isolated drivers") + } + + // write to a file and check it presence in host + r := execTask(t, driver, taskID, `cat /etc/resolv.conf`, + false, "") + require.Zero(t, r.exitCode) + + resolvConf := []byte(strings.TrimSpace(r.stdout)) + + if dns != nil { + if len(dns.Servers) > 0 { + require.ElementsMatch(t, dns.Servers, dresolvconf.GetNameservers(resolvConf, dtypes.IP)) + } + if len(dns.Searches) > 0 { + require.ElementsMatch(t, dns.Searches, dresolvconf.GetSearchDomains(resolvConf)) + } + if len(dns.Options) > 0 { + require.ElementsMatch(t, dns.Options, dresolvconf.GetOptions(resolvConf)) + } + } else { + system, err := dresolvconf.Get() + require.NoError(t, err) + require.ElementsMatch(t, dresolvconf.GetNameservers(system.Content, dtypes.IP), dresolvconf.GetNameservers(resolvConf, dtypes.IP)) + require.ElementsMatch(t, dresolvconf.GetSearchDomains(system.Content), dresolvconf.GetSearchDomains(resolvConf)) + require.ElementsMatch(t, dresolvconf.GetOptions(system.Content), dresolvconf.GetOptions(resolvConf)) + } + }) +} diff --git a/plugins/drivers/utils.go b/plugins/drivers/utils.go index c146e96ef80d..bfb3d5c2614b 100644 --- a/plugins/drivers/utils.go +++ b/plugins/drivers/utils.go @@ -65,6 +65,7 @@ func taskConfigFromProto(pb *proto.TaskConfig) *TaskConfig { StderrPath: pb.StderrPath, AllocID: pb.AllocId, NetworkIsolation: NetworkIsolationSpecFromProto(pb.NetworkIsolationSpec), + DNS: dnsConfigFromProto(pb.Dns), } } @@ -89,6 +90,7 @@ func taskConfigToProto(cfg *TaskConfig) *proto.TaskConfig { StderrPath: cfg.StderrPath, AllocId: cfg.AllocID, NetworkIsolationSpec: NetworkIsolationSpecToProto(cfg.NetworkIsolation), + Dns: dnsConfigToProto(cfg.DNS), } return pb } @@ -625,3 +627,27 @@ func NetworkIsolationSpecFromProto(pb *proto.NetworkIsolationSpec) *NetworkIsola Mode: netIsolationModeFromProto(pb.Mode), } } + +func dnsConfigToProto(dns *DNSConfig) *proto.DNSConfig { + if dns == nil { + return nil + } + + return &proto.DNSConfig{ + Servers: dns.Servers, + Searches: dns.Searches, + Options: dns.Options, + } +} + +func dnsConfigFromProto(pb *proto.DNSConfig) *DNSConfig { + if pb == nil { + return nil + } + + return &DNSConfig{ + Servers: pb.Servers, + Searches: pb.Searches, + Options: pb.Options, + } +} diff --git a/scheduler/feasible.go b/scheduler/feasible.go index 848764443fc1..edcaae1255dd 100644 --- a/scheduler/feasible.go +++ b/scheduler/feasible.go @@ -314,6 +314,44 @@ func (c *CSIVolumeChecker) hasPlugins(n *structs.Node) (bool, string) { return true, "" } +// NetworkChecker is a FeasibilityChecker which returns whether a node has the +// network resources necessary to schedule the task group +type NetworkChecker struct { + ctx Context + networkMode string +} + +func NewNetworkChecker(ctx Context) *NetworkChecker { + return &NetworkChecker{ctx: ctx, networkMode: "host"} +} + +func (c *NetworkChecker) SetNetworkMode(netMode string) { + c.networkMode = netMode +} + +func (c *NetworkChecker) Feasible(option *structs.Node) bool { + if c.hasNetwork(option) { + return true + } + + c.ctx.Metrics().FilterNode(option, "missing network") + return false +} + +func (c *NetworkChecker) hasNetwork(option *structs.Node) bool { + if option.NodeResources == nil { + return false + } + + for _, nw := range option.NodeResources.Networks { + if nw.Mode == c.networkMode { + return true + } + } + + return false +} + // DriverChecker is a FeasibilityChecker which returns whether a node has the // drivers necessary to scheduler a task group. type DriverChecker struct { diff --git a/scheduler/feasible_test.go b/scheduler/feasible_test.go index 18e50c6ee4e7..725a90bc121f 100644 --- a/scheduler/feasible_test.go +++ b/scheduler/feasible_test.go @@ -397,6 +397,48 @@ func TestCSIVolumeChecker(t *testing.T) { } } +func TestNetworkChecker(t *testing.T) { + _, ctx := testContext(t) + nodes := []*structs.Node{ + mock.Node(), + mock.Node(), + mock.Node(), + } + nodes[0].NodeResources.Networks = append(nodes[0].NodeResources.Networks, &structs.NetworkResource{Mode: "bridge"}) + nodes[1].NodeResources.Networks = append(nodes[1].NodeResources.Networks, &structs.NetworkResource{Mode: "bridge"}) + nodes[2].NodeResources.Networks = append(nodes[2].NodeResources.Networks, &structs.NetworkResource{Mode: "cni/mynet"}) + + checker := NewNetworkChecker(ctx) + cases := []struct { + mode string + results []bool + }{ + { + mode: "host", + results: []bool{true, true, true}, + }, + { + mode: "bridge", + results: []bool{true, true, false}, + }, + { + mode: "cni/mynet", + results: []bool{false, false, true}, + }, + { + mode: "cni/nonexistent", + results: []bool{false, false, false}, + }, + } + + for _, c := range cases { + checker.SetNetworkMode(c.mode) + for i, node := range nodes { + require.Equal(t, c.results[i], checker.Feasible(node), "mode=%q, idx=%d", c.mode, i) + } + } +} + func TestDriverChecker_DriverInfo(t *testing.T) { _, ctx := testContext(t) nodes := []*structs.Node{ diff --git a/scheduler/generic_sched_test.go b/scheduler/generic_sched_test.go index 66e32682048e..3390f0f1d42f 100644 --- a/scheduler/generic_sched_test.go +++ b/scheduler/generic_sched_test.go @@ -5204,6 +5204,7 @@ func TestServiceSched_Preemption(t *testing.T) { }, Networks: []*structs.NetworkResource{ { + Mode: "host", Device: "eth0", CIDR: "192.168.0.100/32", MBits: 1000, diff --git a/scheduler/stack.go b/scheduler/stack.go index b381e4c5871d..d1a94bcc3342 100644 --- a/scheduler/stack.go +++ b/scheduler/stack.go @@ -52,6 +52,7 @@ type GenericStack struct { taskGroupDevices *DeviceChecker taskGroupHostVolumes *HostVolumeChecker taskGroupCSIVolumes *CSIVolumeChecker + taskGroupNetwork *NetworkChecker distinctHostsConstraint *DistinctHostsIterator distinctPropertyConstraint *DistinctPropertyIterator @@ -135,6 +136,9 @@ func (s *GenericStack) Select(tg *structs.TaskGroup, options *SelectOptions) *Ra s.taskGroupDevices.SetTaskGroup(tg) s.taskGroupHostVolumes.SetVolumes(tg.Volumes) s.taskGroupCSIVolumes.SetVolumes(tg.Volumes) + if len(tg.Networks) > 0 { + s.taskGroupNetwork.SetNetworkMode(tg.Networks[0].Mode) + } s.distinctHostsConstraint.SetTaskGroup(tg) s.distinctPropertyConstraint.SetTaskGroup(tg) s.wrappedChecks.SetTaskGroup(tg.Name) @@ -332,6 +336,9 @@ func NewGenericStack(batch bool, ctx Context) *GenericStack { // Filter on available, healthy CSI plugins s.taskGroupCSIVolumes = NewCSIVolumeChecker(ctx) + // Filter on available client networks + s.taskGroupNetwork = NewNetworkChecker(ctx) + // Create the feasibility wrapper which wraps all feasibility checks in // which feasibility checking can be skipped if the computed node class has // previously been marked as eligible or ineligible. Generally this will be @@ -340,7 +347,8 @@ func NewGenericStack(batch bool, ctx Context) *GenericStack { tgs := []FeasibilityChecker{s.taskGroupDrivers, s.taskGroupConstraint, s.taskGroupHostVolumes, - s.taskGroupDevices} + s.taskGroupDevices, + s.taskGroupNetwork} avail := []FeasibilityChecker{s.taskGroupCSIVolumes} s.wrappedChecks = NewFeasibilityWrapper(ctx, s.quota, jobs, tgs, avail) diff --git a/scheduler/util.go b/scheduler/util.go index 713de7ad31d9..a3f4565754da 100644 --- a/scheduler/util.go +++ b/scheduler/util.go @@ -445,6 +445,10 @@ func networkUpdated(netA, netB []*structs.NetworkResource) bool { return true } + if !reflect.DeepEqual(an.DNS, bn.DNS) { + return true + } + aPorts, bPorts := networkPortMap(an), networkPortMap(bn) if !reflect.DeepEqual(aPorts, bPorts) { return true diff --git a/vendor/github.com/docker/libnetwork/LICENSE b/vendor/github.com/docker/libnetwork/LICENSE new file mode 100644 index 000000000000..e06d2081865a --- /dev/null +++ b/vendor/github.com/docker/libnetwork/LICENSE @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/vendor/github.com/docker/libnetwork/resolvconf/README.md b/vendor/github.com/docker/libnetwork/resolvconf/README.md new file mode 100644 index 000000000000..cdda554ba572 --- /dev/null +++ b/vendor/github.com/docker/libnetwork/resolvconf/README.md @@ -0,0 +1 @@ +Package resolvconf provides utility code to query and update DNS configuration in /etc/resolv.conf diff --git a/vendor/github.com/docker/libnetwork/resolvconf/dns/resolvconf.go b/vendor/github.com/docker/libnetwork/resolvconf/dns/resolvconf.go new file mode 100644 index 000000000000..e348bc57f56b --- /dev/null +++ b/vendor/github.com/docker/libnetwork/resolvconf/dns/resolvconf.go @@ -0,0 +1,26 @@ +package dns + +import ( + "regexp" +) + +// IPLocalhost is a regex pattern for IPv4 or IPv6 loopback range. +const IPLocalhost = `((127\.([0-9]{1,3}\.){2}[0-9]{1,3})|(::1)$)` + +// IPv4Localhost is a regex pattern for IPv4 localhost address range. +const IPv4Localhost = `(127\.([0-9]{1,3}\.){2}[0-9]{1,3})` + +var localhostIPRegexp = regexp.MustCompile(IPLocalhost) +var localhostIPv4Regexp = regexp.MustCompile(IPv4Localhost) + +// IsLocalhost returns true if ip matches the localhost IP regular expression. +// Used for determining if nameserver settings are being passed which are +// localhost addresses +func IsLocalhost(ip string) bool { + return localhostIPRegexp.MatchString(ip) +} + +// IsIPv4Localhost returns true if ip matches the IPv4 localhost regular expression. +func IsIPv4Localhost(ip string) bool { + return localhostIPv4Regexp.MatchString(ip) +} diff --git a/vendor/github.com/docker/libnetwork/resolvconf/resolvconf.go b/vendor/github.com/docker/libnetwork/resolvconf/resolvconf.go new file mode 100644 index 000000000000..946bb87123e8 --- /dev/null +++ b/vendor/github.com/docker/libnetwork/resolvconf/resolvconf.go @@ -0,0 +1,285 @@ +// Package resolvconf provides utility code to query and update DNS configuration in /etc/resolv.conf +package resolvconf + +import ( + "bytes" + "io/ioutil" + "regexp" + "strings" + "sync" + + "github.com/docker/docker/pkg/ioutils" + "github.com/docker/libnetwork/resolvconf/dns" + "github.com/docker/libnetwork/types" + "github.com/sirupsen/logrus" +) + +const ( + // defaultPath is the default path to the resolv.conf that contains information to resolve DNS. See Path(). + defaultPath = "/etc/resolv.conf" + // alternatePath is a path different from defaultPath, that may be used to resolve DNS. See Path(). + alternatePath = "/run/systemd/resolve/resolv.conf" +) + +var ( + detectSystemdResolvConfOnce sync.Once + pathAfterSystemdDetection = defaultPath +) + +// Path returns the path to the resolv.conf file that libnetwork should use. +// +// When /etc/resolv.conf contains 127.0.0.53 as the only nameserver, then +// it is assumed systemd-resolved manages DNS. Because inside the container 127.0.0.53 +// is not a valid DNS server, Path() returns /run/systemd/resolve/resolv.conf +// which is the resolv.conf that systemd-resolved generates and manages. +// Otherwise Path() returns /etc/resolv.conf. +// +// Errors are silenced as they will inevitably resurface at future open/read calls. +// +// More information at https://www.freedesktop.org/software/systemd/man/systemd-resolved.service.html#/etc/resolv.conf +func Path() string { + detectSystemdResolvConfOnce.Do(func() { + candidateResolvConf, err := ioutil.ReadFile(defaultPath) + if err != nil { + // silencing error as it will resurface at next calls trying to read defaultPath + return + } + ns := GetNameservers(candidateResolvConf, types.IP) + if len(ns) == 1 && ns[0] == "127.0.0.53" { + pathAfterSystemdDetection = alternatePath + logrus.Infof("detected 127.0.0.53 nameserver, assuming systemd-resolved, so using resolv.conf: %s", alternatePath) + } + }) + return pathAfterSystemdDetection +} + +var ( + // Note: the default IPv4 & IPv6 resolvers are set to Google's Public DNS + defaultIPv4Dns = []string{"nameserver 8.8.8.8", "nameserver 8.8.4.4"} + defaultIPv6Dns = []string{"nameserver 2001:4860:4860::8888", "nameserver 2001:4860:4860::8844"} + ipv4NumBlock = `(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)` + ipv4Address = `(` + ipv4NumBlock + `\.){3}` + ipv4NumBlock + // This is not an IPv6 address verifier as it will accept a super-set of IPv6, and also + // will *not match* IPv4-Embedded IPv6 Addresses (RFC6052), but that and other variants + // -- e.g. other link-local types -- either won't work in containers or are unnecessary. + // For readability and sufficiency for Docker purposes this seemed more reasonable than a + // 1000+ character regexp with exact and complete IPv6 validation + ipv6Address = `([0-9A-Fa-f]{0,4}:){2,7}([0-9A-Fa-f]{0,4})(%\w+)?` + + localhostNSRegexp = regexp.MustCompile(`(?m)^nameserver\s+` + dns.IPLocalhost + `\s*\n*`) + nsIPv6Regexp = regexp.MustCompile(`(?m)^nameserver\s+` + ipv6Address + `\s*\n*`) + nsRegexp = regexp.MustCompile(`^\s*nameserver\s*((` + ipv4Address + `)|(` + ipv6Address + `))\s*$`) + nsIPv6Regexpmatch = regexp.MustCompile(`^\s*nameserver\s*((` + ipv6Address + `))\s*$`) + nsIPv4Regexpmatch = regexp.MustCompile(`^\s*nameserver\s*((` + ipv4Address + `))\s*$`) + searchRegexp = regexp.MustCompile(`^\s*search\s*(([^\s]+\s*)*)$`) + optionsRegexp = regexp.MustCompile(`^\s*options\s*(([^\s]+\s*)*)$`) +) + +var lastModified struct { + sync.Mutex + sha256 string + contents []byte +} + +// File contains the resolv.conf content and its hash +type File struct { + Content []byte + Hash string +} + +// Get returns the contents of /etc/resolv.conf and its hash +func Get() (*File, error) { + return GetSpecific(Path()) +} + +// GetSpecific returns the contents of the user specified resolv.conf file and its hash +func GetSpecific(path string) (*File, error) { + resolv, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + hash, err := ioutils.HashData(bytes.NewReader(resolv)) + if err != nil { + return nil, err + } + return &File{Content: resolv, Hash: hash}, nil +} + +// GetIfChanged retrieves the host /etc/resolv.conf file, checks against the last hash +// and, if modified since last check, returns the bytes and new hash. +// This feature is used by the resolv.conf updater for containers +func GetIfChanged() (*File, error) { + lastModified.Lock() + defer lastModified.Unlock() + + resolv, err := ioutil.ReadFile(Path()) + if err != nil { + return nil, err + } + newHash, err := ioutils.HashData(bytes.NewReader(resolv)) + if err != nil { + return nil, err + } + if lastModified.sha256 != newHash { + lastModified.sha256 = newHash + lastModified.contents = resolv + return &File{Content: resolv, Hash: newHash}, nil + } + // nothing changed, so return no data + return nil, nil +} + +// GetLastModified retrieves the last used contents and hash of the host resolv.conf. +// Used by containers updating on restart +func GetLastModified() *File { + lastModified.Lock() + defer lastModified.Unlock() + + return &File{Content: lastModified.contents, Hash: lastModified.sha256} +} + +// FilterResolvDNS cleans up the config in resolvConf. It has two main jobs: +// 1. It looks for localhost (127.*|::1) entries in the provided +// resolv.conf, removing local nameserver entries, and, if the resulting +// cleaned config has no defined nameservers left, adds default DNS entries +// 2. Given the caller provides the enable/disable state of IPv6, the filter +// code will remove all IPv6 nameservers if it is not enabled for containers +// +func FilterResolvDNS(resolvConf []byte, ipv6Enabled bool) (*File, error) { + cleanedResolvConf := localhostNSRegexp.ReplaceAll(resolvConf, []byte{}) + // if IPv6 is not enabled, also clean out any IPv6 address nameserver + if !ipv6Enabled { + cleanedResolvConf = nsIPv6Regexp.ReplaceAll(cleanedResolvConf, []byte{}) + } + // if the resulting resolvConf has no more nameservers defined, add appropriate + // default DNS servers for IPv4 and (optionally) IPv6 + if len(GetNameservers(cleanedResolvConf, types.IP)) == 0 { + logrus.Infof("No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: %v", defaultIPv4Dns) + dns := defaultIPv4Dns + if ipv6Enabled { + logrus.Infof("IPv6 enabled; Adding default IPv6 external servers: %v", defaultIPv6Dns) + dns = append(dns, defaultIPv6Dns...) + } + cleanedResolvConf = append(cleanedResolvConf, []byte("\n"+strings.Join(dns, "\n"))...) + } + hash, err := ioutils.HashData(bytes.NewReader(cleanedResolvConf)) + if err != nil { + return nil, err + } + return &File{Content: cleanedResolvConf, Hash: hash}, nil +} + +// getLines parses input into lines and strips away comments. +func getLines(input []byte, commentMarker []byte) [][]byte { + lines := bytes.Split(input, []byte("\n")) + var output [][]byte + for _, currentLine := range lines { + var commentIndex = bytes.Index(currentLine, commentMarker) + if commentIndex == -1 { + output = append(output, currentLine) + } else { + output = append(output, currentLine[:commentIndex]) + } + } + return output +} + +// GetNameservers returns nameservers (if any) listed in /etc/resolv.conf +func GetNameservers(resolvConf []byte, kind int) []string { + nameservers := []string{} + for _, line := range getLines(resolvConf, []byte("#")) { + var ns [][]byte + if kind == types.IP { + ns = nsRegexp.FindSubmatch(line) + } else if kind == types.IPv4 { + ns = nsIPv4Regexpmatch.FindSubmatch(line) + } else if kind == types.IPv6 { + ns = nsIPv6Regexpmatch.FindSubmatch(line) + } + if len(ns) > 0 { + nameservers = append(nameservers, string(ns[1])) + } + } + return nameservers +} + +// GetNameserversAsCIDR returns nameservers (if any) listed in +// /etc/resolv.conf as CIDR blocks (e.g., "1.2.3.4/32") +// This function's output is intended for net.ParseCIDR +func GetNameserversAsCIDR(resolvConf []byte) []string { + nameservers := []string{} + for _, nameserver := range GetNameservers(resolvConf, types.IP) { + var address string + // If IPv6, strip zone if present + if strings.Contains(nameserver, ":") { + address = strings.Split(nameserver, "%")[0] + "/128" + } else { + address = nameserver + "/32" + } + nameservers = append(nameservers, address) + } + return nameservers +} + +// GetSearchDomains returns search domains (if any) listed in /etc/resolv.conf +// If more than one search line is encountered, only the contents of the last +// one is returned. +func GetSearchDomains(resolvConf []byte) []string { + domains := []string{} + for _, line := range getLines(resolvConf, []byte("#")) { + match := searchRegexp.FindSubmatch(line) + if match == nil { + continue + } + domains = strings.Fields(string(match[1])) + } + return domains +} + +// GetOptions returns options (if any) listed in /etc/resolv.conf +// If more than one options line is encountered, only the contents of the last +// one is returned. +func GetOptions(resolvConf []byte) []string { + options := []string{} + for _, line := range getLines(resolvConf, []byte("#")) { + match := optionsRegexp.FindSubmatch(line) + if match == nil { + continue + } + options = strings.Fields(string(match[1])) + } + return options +} + +// Build writes a configuration file to path containing a "nameserver" entry +// for every element in dns, a "search" entry for every element in +// dnsSearch, and an "options" entry for every element in dnsOptions. +func Build(path string, dns, dnsSearch, dnsOptions []string) (*File, error) { + content := bytes.NewBuffer(nil) + if len(dnsSearch) > 0 { + if searchString := strings.Join(dnsSearch, " "); strings.Trim(searchString, " ") != "." { + if _, err := content.WriteString("search " + searchString + "\n"); err != nil { + return nil, err + } + } + } + for _, dns := range dns { + if _, err := content.WriteString("nameserver " + dns + "\n"); err != nil { + return nil, err + } + } + if len(dnsOptions) > 0 { + if optsString := strings.Join(dnsOptions, " "); strings.Trim(optsString, " ") != "" { + if _, err := content.WriteString("options " + optsString + "\n"); err != nil { + return nil, err + } + } + } + + hash, err := ioutils.HashData(bytes.NewReader(content.Bytes())) + if err != nil { + return nil, err + } + + return &File{Content: content.Bytes(), Hash: hash}, ioutil.WriteFile(path, content.Bytes(), 0644) +} diff --git a/vendor/github.com/docker/libnetwork/types/types.go b/vendor/github.com/docker/libnetwork/types/types.go new file mode 100644 index 000000000000..db1960c104e8 --- /dev/null +++ b/vendor/github.com/docker/libnetwork/types/types.go @@ -0,0 +1,653 @@ +// Package types contains types that are common across libnetwork project +package types + +import ( + "bytes" + "fmt" + "net" + "strconv" + "strings" + + "github.com/ishidawataru/sctp" +) + +// constants for the IP address type +const ( + IP = iota // IPv4 and IPv6 + IPv4 + IPv6 +) + +// EncryptionKey is the libnetwork representation of the key distributed by the lead +// manager. +type EncryptionKey struct { + Subsystem string + Algorithm int32 + Key []byte + LamportTime uint64 +} + +// UUID represents a globally unique ID of various resources like network and endpoint +type UUID string + +// QosPolicy represents a quality of service policy on an endpoint +type QosPolicy struct { + MaxEgressBandwidth uint64 +} + +// TransportPort represents a local Layer 4 endpoint +type TransportPort struct { + Proto Protocol + Port uint16 +} + +// Equal checks if this instance of Transportport is equal to the passed one +func (t *TransportPort) Equal(o *TransportPort) bool { + if t == o { + return true + } + + if o == nil { + return false + } + + if t.Proto != o.Proto || t.Port != o.Port { + return false + } + + return true +} + +// GetCopy returns a copy of this TransportPort structure instance +func (t *TransportPort) GetCopy() TransportPort { + return TransportPort{Proto: t.Proto, Port: t.Port} +} + +// String returns the TransportPort structure in string form +func (t *TransportPort) String() string { + return fmt.Sprintf("%s/%d", t.Proto.String(), t.Port) +} + +// FromString reads the TransportPort structure from string +func (t *TransportPort) FromString(s string) error { + ps := strings.Split(s, "/") + if len(ps) == 2 { + t.Proto = ParseProtocol(ps[0]) + if p, err := strconv.ParseUint(ps[1], 10, 16); err == nil { + t.Port = uint16(p) + return nil + } + } + return BadRequestErrorf("invalid format for transport port: %s", s) +} + +// PortBinding represents a port binding between the container and the host +type PortBinding struct { + Proto Protocol + IP net.IP + Port uint16 + HostIP net.IP + HostPort uint16 + HostPortEnd uint16 +} + +// HostAddr returns the host side transport address +func (p PortBinding) HostAddr() (net.Addr, error) { + switch p.Proto { + case UDP: + return &net.UDPAddr{IP: p.HostIP, Port: int(p.HostPort)}, nil + case TCP: + return &net.TCPAddr{IP: p.HostIP, Port: int(p.HostPort)}, nil + case SCTP: + return &sctp.SCTPAddr{IPAddrs: []net.IPAddr{{IP: p.HostIP}}, Port: int(p.HostPort)}, nil + default: + return nil, ErrInvalidProtocolBinding(p.Proto.String()) + } +} + +// ContainerAddr returns the container side transport address +func (p PortBinding) ContainerAddr() (net.Addr, error) { + switch p.Proto { + case UDP: + return &net.UDPAddr{IP: p.IP, Port: int(p.Port)}, nil + case TCP: + return &net.TCPAddr{IP: p.IP, Port: int(p.Port)}, nil + case SCTP: + return &sctp.SCTPAddr{IPAddrs: []net.IPAddr{{IP: p.IP}}, Port: int(p.Port)}, nil + default: + return nil, ErrInvalidProtocolBinding(p.Proto.String()) + } +} + +// GetCopy returns a copy of this PortBinding structure instance +func (p *PortBinding) GetCopy() PortBinding { + return PortBinding{ + Proto: p.Proto, + IP: GetIPCopy(p.IP), + Port: p.Port, + HostIP: GetIPCopy(p.HostIP), + HostPort: p.HostPort, + HostPortEnd: p.HostPortEnd, + } +} + +// String returns the PortBinding structure in string form +func (p *PortBinding) String() string { + ret := fmt.Sprintf("%s/", p.Proto) + if p.IP != nil { + ret += p.IP.String() + } + ret = fmt.Sprintf("%s:%d/", ret, p.Port) + if p.HostIP != nil { + ret += p.HostIP.String() + } + ret = fmt.Sprintf("%s:%d", ret, p.HostPort) + return ret +} + +// FromString reads the PortBinding structure from string s. +// String s is a triple of "protocol/containerIP:port/hostIP:port" +// containerIP and hostIP can be in dotted decimal ("192.0.2.1") or IPv6 ("2001:db8::68") form. +// Zoned addresses ("169.254.0.23%eth0" or "fe80::1ff:fe23:4567:890a%eth0") are not supported. +// If string s is incorrectly formatted or the IP addresses or ports cannot be parsed, FromString +// returns an error. +func (p *PortBinding) FromString(s string) error { + ps := strings.Split(s, "/") + if len(ps) != 3 { + return BadRequestErrorf("invalid format for port binding: %s", s) + } + + p.Proto = ParseProtocol(ps[0]) + + var err error + if p.IP, p.Port, err = parseIPPort(ps[1]); err != nil { + return BadRequestErrorf("failed to parse Container IP/Port in port binding: %s", err.Error()) + } + + if p.HostIP, p.HostPort, err = parseIPPort(ps[2]); err != nil { + return BadRequestErrorf("failed to parse Host IP/Port in port binding: %s", err.Error()) + } + + return nil +} + +func parseIPPort(s string) (net.IP, uint16, error) { + hoststr, portstr, err := net.SplitHostPort(s) + if err != nil { + return nil, 0, err + } + + ip := net.ParseIP(hoststr) + if ip == nil { + return nil, 0, BadRequestErrorf("invalid ip: %s", hoststr) + } + + port, err := strconv.ParseUint(portstr, 10, 16) + if err != nil { + return nil, 0, BadRequestErrorf("invalid port: %s", portstr) + } + + return ip, uint16(port), nil +} + +// Equal checks if this instance of PortBinding is equal to the passed one +func (p *PortBinding) Equal(o *PortBinding) bool { + if p == o { + return true + } + + if o == nil { + return false + } + + if p.Proto != o.Proto || p.Port != o.Port || + p.HostPort != o.HostPort || p.HostPortEnd != o.HostPortEnd { + return false + } + + if p.IP != nil { + if !p.IP.Equal(o.IP) { + return false + } + } else { + if o.IP != nil { + return false + } + } + + if p.HostIP != nil { + if !p.HostIP.Equal(o.HostIP) { + return false + } + } else { + if o.HostIP != nil { + return false + } + } + + return true +} + +// ErrInvalidProtocolBinding is returned when the port binding protocol is not valid. +type ErrInvalidProtocolBinding string + +func (ipb ErrInvalidProtocolBinding) Error() string { + return fmt.Sprintf("invalid transport protocol: %s", string(ipb)) +} + +const ( + // ICMP is for the ICMP ip protocol + ICMP = 1 + // TCP is for the TCP ip protocol + TCP = 6 + // UDP is for the UDP ip protocol + UDP = 17 + // SCTP is for the SCTP ip protocol + SCTP = 132 +) + +// Protocol represents an IP protocol number +type Protocol uint8 + +func (p Protocol) String() string { + switch p { + case ICMP: + return "icmp" + case TCP: + return "tcp" + case UDP: + return "udp" + case SCTP: + return "sctp" + default: + return fmt.Sprintf("%d", p) + } +} + +// ParseProtocol returns the respective Protocol type for the passed string +func ParseProtocol(s string) Protocol { + switch strings.ToLower(s) { + case "icmp": + return ICMP + case "udp": + return UDP + case "tcp": + return TCP + case "sctp": + return SCTP + default: + return 0 + } +} + +// GetMacCopy returns a copy of the passed MAC address +func GetMacCopy(from net.HardwareAddr) net.HardwareAddr { + if from == nil { + return nil + } + to := make(net.HardwareAddr, len(from)) + copy(to, from) + return to +} + +// GetIPCopy returns a copy of the passed IP address +func GetIPCopy(from net.IP) net.IP { + if from == nil { + return nil + } + to := make(net.IP, len(from)) + copy(to, from) + return to +} + +// GetIPNetCopy returns a copy of the passed IP Network +func GetIPNetCopy(from *net.IPNet) *net.IPNet { + if from == nil { + return nil + } + bm := make(net.IPMask, len(from.Mask)) + copy(bm, from.Mask) + return &net.IPNet{IP: GetIPCopy(from.IP), Mask: bm} +} + +// GetIPNetCanonical returns the canonical form for the passed network +func GetIPNetCanonical(nw *net.IPNet) *net.IPNet { + if nw == nil { + return nil + } + c := GetIPNetCopy(nw) + c.IP = c.IP.Mask(nw.Mask) + return c +} + +// CompareIPNet returns equal if the two IP Networks are equal +func CompareIPNet(a, b *net.IPNet) bool { + if a == b { + return true + } + if a == nil || b == nil { + return false + } + return a.IP.Equal(b.IP) && bytes.Equal(a.Mask, b.Mask) +} + +// GetMinimalIP returns the address in its shortest form +// If ip contains an IPv4-mapped IPv6 address, the 4-octet form of the IPv4 address will be returned. +// Otherwise ip is returned unchanged. +func GetMinimalIP(ip net.IP) net.IP { + if ip != nil && ip.To4() != nil { + return ip.To4() + } + return ip +} + +// GetMinimalIPNet returns a copy of the passed IP Network with congruent ip and mask notation +func GetMinimalIPNet(nw *net.IPNet) *net.IPNet { + if nw == nil { + return nil + } + if len(nw.IP) == 16 && nw.IP.To4() != nil { + m := nw.Mask + if len(m) == 16 { + m = m[12:16] + } + return &net.IPNet{IP: nw.IP.To4(), Mask: m} + } + return nw +} + +// IsIPNetValid returns true if the ipnet is a valid network/mask +// combination. Otherwise returns false. +func IsIPNetValid(nw *net.IPNet) bool { + return nw.String() != "0.0.0.0/0" +} + +var v4inV6MaskPrefix = []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} + +// compareIPMask checks if the passed ip and mask are semantically compatible. +// It returns the byte indexes for the address and mask so that caller can +// do bitwise operations without modifying address representation. +func compareIPMask(ip net.IP, mask net.IPMask) (is int, ms int, err error) { + // Find the effective starting of address and mask + if len(ip) == net.IPv6len && ip.To4() != nil { + is = 12 + } + if len(ip[is:]) == net.IPv4len && len(mask) == net.IPv6len && bytes.Equal(mask[:12], v4inV6MaskPrefix) { + ms = 12 + } + // Check if address and mask are semantically compatible + if len(ip[is:]) != len(mask[ms:]) { + err = fmt.Errorf("ip and mask are not compatible: (%#v, %#v)", ip, mask) + } + return +} + +// GetHostPartIP returns the host portion of the ip address identified by the mask. +// IP address representation is not modified. If address and mask are not compatible +// an error is returned. +func GetHostPartIP(ip net.IP, mask net.IPMask) (net.IP, error) { + // Find the effective starting of address and mask + is, ms, err := compareIPMask(ip, mask) + if err != nil { + return nil, fmt.Errorf("cannot compute host portion ip address because %s", err) + } + + // Compute host portion + out := GetIPCopy(ip) + for i := 0; i < len(mask[ms:]); i++ { + out[is+i] &= ^mask[ms+i] + } + + return out, nil +} + +// GetBroadcastIP returns the broadcast ip address for the passed network (ip and mask). +// IP address representation is not modified. If address and mask are not compatible +// an error is returned. +func GetBroadcastIP(ip net.IP, mask net.IPMask) (net.IP, error) { + // Find the effective starting of address and mask + is, ms, err := compareIPMask(ip, mask) + if err != nil { + return nil, fmt.Errorf("cannot compute broadcast ip address because %s", err) + } + + // Compute broadcast address + out := GetIPCopy(ip) + for i := 0; i < len(mask[ms:]); i++ { + out[is+i] |= ^mask[ms+i] + } + + return out, nil +} + +// ParseCIDR returns the *net.IPNet represented by the passed CIDR notation +func ParseCIDR(cidr string) (n *net.IPNet, e error) { + var i net.IP + if i, n, e = net.ParseCIDR(cidr); e == nil { + n.IP = i + } + return +} + +const ( + // NEXTHOP indicates a StaticRoute with an IP next hop. + NEXTHOP = iota + + // CONNECTED indicates a StaticRoute with an interface for directly connected peers. + CONNECTED +) + +// StaticRoute is a statically-provisioned IP route. +type StaticRoute struct { + Destination *net.IPNet + + RouteType int // NEXT_HOP or CONNECTED + + // NextHop will be resolved by the kernel (i.e. as a loose hop). + NextHop net.IP +} + +// GetCopy returns a copy of this StaticRoute structure +func (r *StaticRoute) GetCopy() *StaticRoute { + d := GetIPNetCopy(r.Destination) + nh := GetIPCopy(r.NextHop) + return &StaticRoute{Destination: d, + RouteType: r.RouteType, + NextHop: nh, + } +} + +// InterfaceStatistics represents the interface's statistics +type InterfaceStatistics struct { + RxBytes uint64 + RxPackets uint64 + RxErrors uint64 + RxDropped uint64 + TxBytes uint64 + TxPackets uint64 + TxErrors uint64 + TxDropped uint64 +} + +func (is *InterfaceStatistics) String() string { + return fmt.Sprintf("\nRxBytes: %d, RxPackets: %d, RxErrors: %d, RxDropped: %d, TxBytes: %d, TxPackets: %d, TxErrors: %d, TxDropped: %d", + is.RxBytes, is.RxPackets, is.RxErrors, is.RxDropped, is.TxBytes, is.TxPackets, is.TxErrors, is.TxDropped) +} + +/****************************** + * Well-known Error Interfaces + ******************************/ + +// MaskableError is an interface for errors which can be ignored by caller +type MaskableError interface { + // Maskable makes implementer into MaskableError type + Maskable() +} + +// RetryError is an interface for errors which might get resolved through retry +type RetryError interface { + // Retry makes implementer into RetryError type + Retry() +} + +// BadRequestError is an interface for errors originated by a bad request +type BadRequestError interface { + // BadRequest makes implementer into BadRequestError type + BadRequest() +} + +// NotFoundError is an interface for errors raised because a needed resource is not available +type NotFoundError interface { + // NotFound makes implementer into NotFoundError type + NotFound() +} + +// ForbiddenError is an interface for errors which denote a valid request that cannot be honored +type ForbiddenError interface { + // Forbidden makes implementer into ForbiddenError type + Forbidden() +} + +// NoServiceError is an interface for errors returned when the required service is not available +type NoServiceError interface { + // NoService makes implementer into NoServiceError type + NoService() +} + +// TimeoutError is an interface for errors raised because of timeout +type TimeoutError interface { + // Timeout makes implementer into TimeoutError type + Timeout() +} + +// NotImplementedError is an interface for errors raised because of requested functionality is not yet implemented +type NotImplementedError interface { + // NotImplemented makes implementer into NotImplementedError type + NotImplemented() +} + +// InternalError is an interface for errors raised because of an internal error +type InternalError interface { + // Internal makes implementer into InternalError type + Internal() +} + +/****************************** + * Well-known Error Formatters + ******************************/ + +// BadRequestErrorf creates an instance of BadRequestError +func BadRequestErrorf(format string, params ...interface{}) error { + return badRequest(fmt.Sprintf(format, params...)) +} + +// NotFoundErrorf creates an instance of NotFoundError +func NotFoundErrorf(format string, params ...interface{}) error { + return notFound(fmt.Sprintf(format, params...)) +} + +// ForbiddenErrorf creates an instance of ForbiddenError +func ForbiddenErrorf(format string, params ...interface{}) error { + return forbidden(fmt.Sprintf(format, params...)) +} + +// NoServiceErrorf creates an instance of NoServiceError +func NoServiceErrorf(format string, params ...interface{}) error { + return noService(fmt.Sprintf(format, params...)) +} + +// NotImplementedErrorf creates an instance of NotImplementedError +func NotImplementedErrorf(format string, params ...interface{}) error { + return notImpl(fmt.Sprintf(format, params...)) +} + +// TimeoutErrorf creates an instance of TimeoutError +func TimeoutErrorf(format string, params ...interface{}) error { + return timeout(fmt.Sprintf(format, params...)) +} + +// InternalErrorf creates an instance of InternalError +func InternalErrorf(format string, params ...interface{}) error { + return internal(fmt.Sprintf(format, params...)) +} + +// InternalMaskableErrorf creates an instance of InternalError and MaskableError +func InternalMaskableErrorf(format string, params ...interface{}) error { + return maskInternal(fmt.Sprintf(format, params...)) +} + +// RetryErrorf creates an instance of RetryError +func RetryErrorf(format string, params ...interface{}) error { + return retry(fmt.Sprintf(format, params...)) +} + +/*********************** + * Internal Error Types + ***********************/ +type badRequest string + +func (br badRequest) Error() string { + return string(br) +} +func (br badRequest) BadRequest() {} + +type maskBadRequest string + +type notFound string + +func (nf notFound) Error() string { + return string(nf) +} +func (nf notFound) NotFound() {} + +type forbidden string + +func (frb forbidden) Error() string { + return string(frb) +} +func (frb forbidden) Forbidden() {} + +type noService string + +func (ns noService) Error() string { + return string(ns) +} +func (ns noService) NoService() {} + +type maskNoService string + +type timeout string + +func (to timeout) Error() string { + return string(to) +} +func (to timeout) Timeout() {} + +type notImpl string + +func (ni notImpl) Error() string { + return string(ni) +} +func (ni notImpl) NotImplemented() {} + +type internal string + +func (nt internal) Error() string { + return string(nt) +} +func (nt internal) Internal() {} + +type maskInternal string + +func (mnt maskInternal) Error() string { + return string(mnt) +} +func (mnt maskInternal) Internal() {} +func (mnt maskInternal) Maskable() {} + +type retry string + +func (r retry) Error() string { + return string(r) +} +func (r retry) Retry() {} diff --git a/vendor/github.com/hashicorp/nomad/api/resources.go b/vendor/github.com/hashicorp/nomad/api/resources.go index cbd24c4bf813..fe5fb521538e 100644 --- a/vendor/github.com/hashicorp/nomad/api/resources.go +++ b/vendor/github.com/hashicorp/nomad/api/resources.go @@ -89,6 +89,12 @@ type Port struct { To int `mapstructure:"to"` } +type DNSConfig struct { + Servers []string `mapstructure:"servers"` + Searches []string `mapstructure:"searches"` + Options []string `mapstructure:"options"` +} + // NetworkResource is used to describe required network // resources of a given task. type NetworkResource struct { @@ -97,6 +103,7 @@ type NetworkResource struct { CIDR string IP string MBits *int + DNS *DNSConfig ReservedPorts []Port DynamicPorts []Port } diff --git a/vendor/github.com/ishidawataru/sctp/.gitignore b/vendor/github.com/ishidawataru/sctp/.gitignore new file mode 100644 index 000000000000..cf2d826c15f2 --- /dev/null +++ b/vendor/github.com/ishidawataru/sctp/.gitignore @@ -0,0 +1,16 @@ +# Binaries for programs and plugins +*.exe +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 +.glide/ + +example/example diff --git a/vendor/github.com/ishidawataru/sctp/.travis.yml b/vendor/github.com/ishidawataru/sctp/.travis.yml new file mode 100644 index 000000000000..01a76be9a44d --- /dev/null +++ b/vendor/github.com/ishidawataru/sctp/.travis.yml @@ -0,0 +1,18 @@ +language: go +go: + - 1.9.x + - 1.10.x + - 1.11.x + - 1.12.x + - 1.13.x + +script: + - go test -v -race ./... + - GOOS=linux GOARCH=amd64 go build . + - GOOS=linux GOARCH=arm go build . + - GOOS=linux GOARCH=arm64 go build . + - GOOS=linux GOARCH=ppc64le go build . + - (go version | grep go1.6 > /dev/null) || GOOS=linux GOARCH=s390x go build . +# can be compiled but not functional: + - GOOS=linux GOARCH=386 go build . + - GOOS=windows GOARCH=amd64 go build . diff --git a/vendor/github.com/ishidawataru/sctp/GO_LICENSE b/vendor/github.com/ishidawataru/sctp/GO_LICENSE new file mode 100644 index 000000000000..6a66aea5eafe --- /dev/null +++ b/vendor/github.com/ishidawataru/sctp/GO_LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/ishidawataru/sctp/LICENSE b/vendor/github.com/ishidawataru/sctp/LICENSE new file mode 100644 index 000000000000..8dada3edaf50 --- /dev/null +++ b/vendor/github.com/ishidawataru/sctp/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/ishidawataru/sctp/NOTICE b/vendor/github.com/ishidawataru/sctp/NOTICE new file mode 100644 index 000000000000..cfb675fd4ba4 --- /dev/null +++ b/vendor/github.com/ishidawataru/sctp/NOTICE @@ -0,0 +1,3 @@ +This source code includes following third party code + +- ipsock_linux.go : licensed by the Go authors, see GO_LICENSE file for the license which applies to the code diff --git a/vendor/github.com/ishidawataru/sctp/README.md b/vendor/github.com/ishidawataru/sctp/README.md new file mode 100644 index 000000000000..574ececa8626 --- /dev/null +++ b/vendor/github.com/ishidawataru/sctp/README.md @@ -0,0 +1,18 @@ +Stream Control Transmission Protocol (SCTP) +---- + +[![Build Status](https://travis-ci.org/ishidawataru/sctp.svg?branch=master)](https://travis-ci.org/ishidawataru/sctp/builds) + +Examples +---- + +See `example/sctp.go` + +```go +$ cd example +$ go build +$ # run example SCTP server +$ ./example -server -port 1000 -ip 10.10.0.1,10.20.0.1 +$ # run example SCTP client +$ ./example -port 1000 -ip 10.10.0.1,10.20.0.1 +``` diff --git a/vendor/github.com/ishidawataru/sctp/go.mod b/vendor/github.com/ishidawataru/sctp/go.mod new file mode 100644 index 000000000000..5adf982b086c --- /dev/null +++ b/vendor/github.com/ishidawataru/sctp/go.mod @@ -0,0 +1,3 @@ +module github.com/ishidawataru/sctp + +go 1.12 diff --git a/vendor/github.com/ishidawataru/sctp/ipsock_linux.go b/vendor/github.com/ishidawataru/sctp/ipsock_linux.go new file mode 100644 index 000000000000..3df30fa4601a --- /dev/null +++ b/vendor/github.com/ishidawataru/sctp/ipsock_linux.go @@ -0,0 +1,222 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the GO_LICENSE file. + +package sctp + +import ( + "net" + "os" + "sync" + "syscall" +) + +//from https://github.com/golang/go +// Boolean to int. +func boolint(b bool) int { + if b { + return 1 + } + return 0 +} + +//from https://github.com/golang/go +func ipToSockaddr(family int, ip net.IP, port int, zone string) (syscall.Sockaddr, error) { + switch family { + case syscall.AF_INET: + if len(ip) == 0 { + ip = net.IPv4zero + } + ip4 := ip.To4() + if ip4 == nil { + return nil, &net.AddrError{Err: "non-IPv4 address", Addr: ip.String()} + } + sa := &syscall.SockaddrInet4{Port: port} + copy(sa.Addr[:], ip4) + return sa, nil + case syscall.AF_INET6: + // In general, an IP wildcard address, which is either + // "0.0.0.0" or "::", means the entire IP addressing + // space. For some historical reason, it is used to + // specify "any available address" on some operations + // of IP node. + // + // When the IP node supports IPv4-mapped IPv6 address, + // we allow an listener to listen to the wildcard + // address of both IP addressing spaces by specifying + // IPv6 wildcard address. + if len(ip) == 0 || ip.Equal(net.IPv4zero) { + ip = net.IPv6zero + } + // We accept any IPv6 address including IPv4-mapped + // IPv6 address. + ip6 := ip.To16() + if ip6 == nil { + return nil, &net.AddrError{Err: "non-IPv6 address", Addr: ip.String()} + } + //we set ZoneId to 0, as currently we use this functon only to probe the IP capabilities of the host + //if real Zone handling is required, the zone cache implementation in golang/net should be pulled here + sa := &syscall.SockaddrInet6{Port: port, ZoneId: 0} + copy(sa.Addr[:], ip6) + return sa, nil + } + return nil, &net.AddrError{Err: "invalid address family", Addr: ip.String()} +} + +//from https://github.com/golang/go +func sockaddr(a *net.TCPAddr, family int) (syscall.Sockaddr, error) { + if a == nil { + return nil, nil + } + return ipToSockaddr(family, a.IP, a.Port, a.Zone) +} + +//from https://github.com/golang/go +type ipStackCapabilities struct { + sync.Once // guards following + ipv4Enabled bool + ipv6Enabled bool + ipv4MappedIPv6Enabled bool +} + +//from https://github.com/golang/go +var ipStackCaps ipStackCapabilities + +//from https://github.com/golang/go +// supportsIPv4 reports whether the platform supports IPv4 networking +// functionality. +func supportsIPv4() bool { + ipStackCaps.Once.Do(ipStackCaps.probe) + return ipStackCaps.ipv4Enabled +} + +//from https://github.com/golang/go +// supportsIPv6 reports whether the platform supports IPv6 networking +// functionality. +func supportsIPv6() bool { + ipStackCaps.Once.Do(ipStackCaps.probe) + return ipStackCaps.ipv6Enabled +} + +//from https://github.com/golang/go +// supportsIPv4map reports whether the platform supports mapping an +// IPv4 address inside an IPv6 address at transport layer +// protocols. See RFC 4291, RFC 4038 and RFC 3493. +func supportsIPv4map() bool { + ipStackCaps.Once.Do(ipStackCaps.probe) + return ipStackCaps.ipv4MappedIPv6Enabled +} + +//from https://github.com/golang/go +// Probe probes IPv4, IPv6 and IPv4-mapped IPv6 communication +// capabilities which are controlled by the IPV6_V6ONLY socket option +// and kernel configuration. +// +// Should we try to use the IPv4 socket interface if we're only +// dealing with IPv4 sockets? As long as the host system understands +// IPv4-mapped IPv6, it's okay to pass IPv4-mapeed IPv6 addresses to +// the IPv6 interface. That simplifies our code and is most +// general. Unfortunately, we need to run on kernels built without +// IPv6 support too. So probe the kernel to figure it out. +func (p *ipStackCapabilities) probe() { + s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) + switch err { + case syscall.EAFNOSUPPORT, syscall.EPROTONOSUPPORT: + case nil: + syscall.Close(s) + p.ipv4Enabled = true + } + var probes = []struct { + laddr net.TCPAddr + value int + }{ + // IPv6 communication capability + {laddr: net.TCPAddr{IP: net.IPv6loopback}, value: 1}, + // IPv4-mapped IPv6 address communication capability + {laddr: net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)}, value: 0}, + } + + for i := range probes { + s, err := syscall.Socket(syscall.AF_INET6, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) + if err != nil { + continue + } + defer syscall.Close(s) + syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, probes[i].value) + sa, err := sockaddr(&(probes[i].laddr), syscall.AF_INET6) + if err != nil { + continue + } + if err := syscall.Bind(s, sa); err != nil { + continue + } + if i == 0 { + p.ipv6Enabled = true + } else { + p.ipv4MappedIPv6Enabled = true + } + } +} + +//from https://github.com/golang/go +//Change: we check the first IP address in the list of candidate SCTP IP addresses +func (a *SCTPAddr) isWildcard() bool { + if a == nil { + return true + } + if 0 == len(a.IPAddrs) { + return true + } + + return a.IPAddrs[0].IP.IsUnspecified() +} + +func (a *SCTPAddr) family() int { + if a != nil { + for _, ip := range a.IPAddrs { + if ip.IP.To4() == nil { + return syscall.AF_INET6 + } + } + } + return syscall.AF_INET +} + +//from https://github.com/golang/go +func favoriteAddrFamily(network string, laddr *SCTPAddr, raddr *SCTPAddr, mode string) (family int, ipv6only bool) { + switch network[len(network)-1] { + case '4': + return syscall.AF_INET, false + case '6': + return syscall.AF_INET6, true + } + + if mode == "listen" && (laddr == nil || laddr.isWildcard()) { + if supportsIPv4map() || !supportsIPv4() { + return syscall.AF_INET6, false + } + if laddr == nil { + return syscall.AF_INET, false + } + return laddr.family(), false + } + + if (laddr == nil || laddr.family() == syscall.AF_INET) && + (raddr == nil || raddr.family() == syscall.AF_INET) { + return syscall.AF_INET, false + } + return syscall.AF_INET6, false +} + +//from https://github.com/golang/go +//Changes: it is for SCTP only +func setDefaultSockopts(s int, family int, ipv6only bool) error { + if family == syscall.AF_INET6 { + // Allow both IP versions even if the OS default + // is otherwise. Note that some operating systems + // never admit this option. + syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, boolint(ipv6only)) + } + // Allow broadcast. + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1)) +} diff --git a/vendor/github.com/ishidawataru/sctp/sctp.go b/vendor/github.com/ishidawataru/sctp/sctp.go new file mode 100644 index 000000000000..94842f42702f --- /dev/null +++ b/vendor/github.com/ishidawataru/sctp/sctp.go @@ -0,0 +1,729 @@ +// Copyright 2019 Wataru Ishida. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sctp + +import ( + "bytes" + "encoding/binary" + "fmt" + "net" + "strconv" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" + "unsafe" +) + +const ( + SOL_SCTP = 132 + + SCTP_BINDX_ADD_ADDR = 0x01 + SCTP_BINDX_REM_ADDR = 0x02 + + MSG_NOTIFICATION = 0x8000 +) + +const ( + SCTP_RTOINFO = iota + SCTP_ASSOCINFO + SCTP_INITMSG + SCTP_NODELAY + SCTP_AUTOCLOSE + SCTP_SET_PEER_PRIMARY_ADDR + SCTP_PRIMARY_ADDR + SCTP_ADAPTATION_LAYER + SCTP_DISABLE_FRAGMENTS + SCTP_PEER_ADDR_PARAMS + SCTP_DEFAULT_SENT_PARAM + SCTP_EVENTS + SCTP_I_WANT_MAPPED_V4_ADDR + SCTP_MAXSEG + SCTP_STATUS + SCTP_GET_PEER_ADDR_INFO + SCTP_DELAYED_ACK_TIME + SCTP_DELAYED_ACK = SCTP_DELAYED_ACK_TIME + SCTP_DELAYED_SACK = SCTP_DELAYED_ACK_TIME + + SCTP_SOCKOPT_BINDX_ADD = 100 + SCTP_SOCKOPT_BINDX_REM = 101 + SCTP_SOCKOPT_PEELOFF = 102 + SCTP_GET_PEER_ADDRS = 108 + SCTP_GET_LOCAL_ADDRS = 109 + SCTP_SOCKOPT_CONNECTX = 110 + SCTP_SOCKOPT_CONNECTX3 = 111 +) + +const ( + SCTP_EVENT_DATA_IO = 1 << iota + SCTP_EVENT_ASSOCIATION + SCTP_EVENT_ADDRESS + SCTP_EVENT_SEND_FAILURE + SCTP_EVENT_PEER_ERROR + SCTP_EVENT_SHUTDOWN + SCTP_EVENT_PARTIAL_DELIVERY + SCTP_EVENT_ADAPTATION_LAYER + SCTP_EVENT_AUTHENTICATION + SCTP_EVENT_SENDER_DRY + + SCTP_EVENT_ALL = SCTP_EVENT_DATA_IO | SCTP_EVENT_ASSOCIATION | SCTP_EVENT_ADDRESS | SCTP_EVENT_SEND_FAILURE | SCTP_EVENT_PEER_ERROR | SCTP_EVENT_SHUTDOWN | SCTP_EVENT_PARTIAL_DELIVERY | SCTP_EVENT_ADAPTATION_LAYER | SCTP_EVENT_AUTHENTICATION | SCTP_EVENT_SENDER_DRY +) + +type SCTPNotificationType int + +const ( + SCTP_SN_TYPE_BASE = SCTPNotificationType(iota + (1 << 15)) + SCTP_ASSOC_CHANGE + SCTP_PEER_ADDR_CHANGE + SCTP_SEND_FAILED + SCTP_REMOTE_ERROR + SCTP_SHUTDOWN_EVENT + SCTP_PARTIAL_DELIVERY_EVENT + SCTP_ADAPTATION_INDICATION + SCTP_AUTHENTICATION_INDICATION + SCTP_SENDER_DRY_EVENT +) + +type NotificationHandler func([]byte) error + +type EventSubscribe struct { + DataIO uint8 + Association uint8 + Address uint8 + SendFailure uint8 + PeerError uint8 + Shutdown uint8 + PartialDelivery uint8 + AdaptationLayer uint8 + Authentication uint8 + SenderDry uint8 +} + +const ( + SCTP_CMSG_INIT = iota + SCTP_CMSG_SNDRCV + SCTP_CMSG_SNDINFO + SCTP_CMSG_RCVINFO + SCTP_CMSG_NXTINFO +) + +const ( + SCTP_UNORDERED = 1 << iota + SCTP_ADDR_OVER + SCTP_ABORT + SCTP_SACK_IMMEDIATELY + SCTP_EOF +) + +const ( + SCTP_MAX_STREAM = 0xffff +) + +type InitMsg struct { + NumOstreams uint16 + MaxInstreams uint16 + MaxAttempts uint16 + MaxInitTimeout uint16 +} + +type SndRcvInfo struct { + Stream uint16 + SSN uint16 + Flags uint16 + _ uint16 + PPID uint32 + Context uint32 + TTL uint32 + TSN uint32 + CumTSN uint32 + AssocID int32 +} + +type SndInfo struct { + SID uint16 + Flags uint16 + PPID uint32 + Context uint32 + AssocID int32 +} + +type GetAddrsOld struct { + AssocID int32 + AddrNum int32 + Addrs uintptr +} + +type NotificationHeader struct { + Type uint16 + Flags uint16 + Length uint32 +} + +type SCTPState uint16 + +const ( + SCTP_COMM_UP = SCTPState(iota) + SCTP_COMM_LOST + SCTP_RESTART + SCTP_SHUTDOWN_COMP + SCTP_CANT_STR_ASSOC +) + +var nativeEndian binary.ByteOrder +var sndRcvInfoSize uintptr + +func init() { + i := uint16(1) + if *(*byte)(unsafe.Pointer(&i)) == 0 { + nativeEndian = binary.BigEndian + } else { + nativeEndian = binary.LittleEndian + } + info := SndRcvInfo{} + sndRcvInfoSize = unsafe.Sizeof(info) +} + +func toBuf(v interface{}) []byte { + var buf bytes.Buffer + binary.Write(&buf, nativeEndian, v) + return buf.Bytes() +} + +func htons(h uint16) uint16 { + if nativeEndian == binary.LittleEndian { + return (h << 8 & 0xff00) | (h >> 8 & 0xff) + } + return h +} + +var ntohs = htons + +// setInitOpts sets options for an SCTP association initialization +// see https://tools.ietf.org/html/rfc4960#page-25 +func setInitOpts(fd int, options InitMsg) error { + optlen := unsafe.Sizeof(options) + _, _, err := setsockopt(fd, SCTP_INITMSG, uintptr(unsafe.Pointer(&options)), uintptr(optlen)) + return err +} + +func setNumOstreams(fd, num int) error { + return setInitOpts(fd, InitMsg{NumOstreams: uint16(num)}) +} + +type SCTPAddr struct { + IPAddrs []net.IPAddr + Port int +} + +func (a *SCTPAddr) ToRawSockAddrBuf() []byte { + p := htons(uint16(a.Port)) + if len(a.IPAddrs) == 0 { // if a.IPAddrs list is empty - fall back to IPv4 zero addr + s := syscall.RawSockaddrInet4{ + Family: syscall.AF_INET, + Port: p, + } + copy(s.Addr[:], net.IPv4zero) + return toBuf(s) + } + buf := []byte{} + for _, ip := range a.IPAddrs { + ipBytes := ip.IP + if len(ipBytes) == 0 { + ipBytes = net.IPv4zero + } + if ip4 := ipBytes.To4(); ip4 != nil { + s := syscall.RawSockaddrInet4{ + Family: syscall.AF_INET, + Port: p, + } + copy(s.Addr[:], ip4) + buf = append(buf, toBuf(s)...) + } else { + var scopeid uint32 + ifi, err := net.InterfaceByName(ip.Zone) + if err == nil { + scopeid = uint32(ifi.Index) + } + s := syscall.RawSockaddrInet6{ + Family: syscall.AF_INET6, + Port: p, + Scope_id: scopeid, + } + copy(s.Addr[:], ipBytes) + buf = append(buf, toBuf(s)...) + } + } + return buf +} + +func (a *SCTPAddr) String() string { + var b bytes.Buffer + + for n, i := range a.IPAddrs { + if i.IP.To4() != nil { + b.WriteString(i.String()) + } else if i.IP.To16() != nil { + b.WriteRune('[') + b.WriteString(i.String()) + b.WriteRune(']') + } + if n < len(a.IPAddrs)-1 { + b.WriteRune('/') + } + } + b.WriteRune(':') + b.WriteString(strconv.Itoa(a.Port)) + return b.String() +} + +func (a *SCTPAddr) Network() string { return "sctp" } + +func ResolveSCTPAddr(network, addrs string) (*SCTPAddr, error) { + tcpnet := "" + switch network { + case "", "sctp": + tcpnet = "tcp" + case "sctp4": + tcpnet = "tcp4" + case "sctp6": + tcpnet = "tcp6" + default: + return nil, fmt.Errorf("invalid net: %s", network) + } + elems := strings.Split(addrs, "/") + if len(elems) == 0 { + return nil, fmt.Errorf("invalid input: %s", addrs) + } + ipaddrs := make([]net.IPAddr, 0, len(elems)) + for _, e := range elems[:len(elems)-1] { + tcpa, err := net.ResolveTCPAddr(tcpnet, e+":") + if err != nil { + return nil, err + } + ipaddrs = append(ipaddrs, net.IPAddr{IP: tcpa.IP, Zone: tcpa.Zone}) + } + tcpa, err := net.ResolveTCPAddr(tcpnet, elems[len(elems)-1]) + if err != nil { + return nil, err + } + if tcpa.IP != nil { + ipaddrs = append(ipaddrs, net.IPAddr{IP: tcpa.IP, Zone: tcpa.Zone}) + } else { + ipaddrs = nil + } + return &SCTPAddr{ + IPAddrs: ipaddrs, + Port: tcpa.Port, + }, nil +} + +func SCTPConnect(fd int, addr *SCTPAddr) (int, error) { + buf := addr.ToRawSockAddrBuf() + param := GetAddrsOld{ + AddrNum: int32(len(buf)), + Addrs: uintptr(uintptr(unsafe.Pointer(&buf[0]))), + } + optlen := unsafe.Sizeof(param) + _, _, err := getsockopt(fd, SCTP_SOCKOPT_CONNECTX3, uintptr(unsafe.Pointer(¶m)), uintptr(unsafe.Pointer(&optlen))) + if err == nil { + return int(param.AssocID), nil + } else if err != syscall.ENOPROTOOPT { + return 0, err + } + r0, _, err := setsockopt(fd, SCTP_SOCKOPT_CONNECTX, uintptr(unsafe.Pointer(&buf[0])), uintptr(len(buf))) + return int(r0), err +} + +func SCTPBind(fd int, addr *SCTPAddr, flags int) error { + var option uintptr + switch flags { + case SCTP_BINDX_ADD_ADDR: + option = SCTP_SOCKOPT_BINDX_ADD + case SCTP_BINDX_REM_ADDR: + option = SCTP_SOCKOPT_BINDX_REM + default: + return syscall.EINVAL + } + + buf := addr.ToRawSockAddrBuf() + _, _, err := setsockopt(fd, option, uintptr(unsafe.Pointer(&buf[0])), uintptr(len(buf))) + return err +} + +type SCTPConn struct { + _fd int32 + notificationHandler NotificationHandler +} + +func (c *SCTPConn) fd() int { + return int(atomic.LoadInt32(&c._fd)) +} + +func NewSCTPConn(fd int, handler NotificationHandler) *SCTPConn { + conn := &SCTPConn{ + _fd: int32(fd), + notificationHandler: handler, + } + return conn +} + +func (c *SCTPConn) Write(b []byte) (int, error) { + return c.SCTPWrite(b, nil) +} + +func (c *SCTPConn) Read(b []byte) (int, error) { + n, _, err := c.SCTPRead(b) + if n < 0 { + n = 0 + } + return n, err +} + +func (c *SCTPConn) SetInitMsg(numOstreams, maxInstreams, maxAttempts, maxInitTimeout int) error { + return setInitOpts(c.fd(), InitMsg{ + NumOstreams: uint16(numOstreams), + MaxInstreams: uint16(maxInstreams), + MaxAttempts: uint16(maxAttempts), + MaxInitTimeout: uint16(maxInitTimeout), + }) +} + +func (c *SCTPConn) SubscribeEvents(flags int) error { + var d, a, ad, sf, p, sh, pa, ada, au, se uint8 + if flags&SCTP_EVENT_DATA_IO > 0 { + d = 1 + } + if flags&SCTP_EVENT_ASSOCIATION > 0 { + a = 1 + } + if flags&SCTP_EVENT_ADDRESS > 0 { + ad = 1 + } + if flags&SCTP_EVENT_SEND_FAILURE > 0 { + sf = 1 + } + if flags&SCTP_EVENT_PEER_ERROR > 0 { + p = 1 + } + if flags&SCTP_EVENT_SHUTDOWN > 0 { + sh = 1 + } + if flags&SCTP_EVENT_PARTIAL_DELIVERY > 0 { + pa = 1 + } + if flags&SCTP_EVENT_ADAPTATION_LAYER > 0 { + ada = 1 + } + if flags&SCTP_EVENT_AUTHENTICATION > 0 { + au = 1 + } + if flags&SCTP_EVENT_SENDER_DRY > 0 { + se = 1 + } + param := EventSubscribe{ + DataIO: d, + Association: a, + Address: ad, + SendFailure: sf, + PeerError: p, + Shutdown: sh, + PartialDelivery: pa, + AdaptationLayer: ada, + Authentication: au, + SenderDry: se, + } + optlen := unsafe.Sizeof(param) + _, _, err := setsockopt(c.fd(), SCTP_EVENTS, uintptr(unsafe.Pointer(¶m)), uintptr(optlen)) + return err +} + +func (c *SCTPConn) SubscribedEvents() (int, error) { + param := EventSubscribe{} + optlen := unsafe.Sizeof(param) + _, _, err := getsockopt(c.fd(), SCTP_EVENTS, uintptr(unsafe.Pointer(¶m)), uintptr(unsafe.Pointer(&optlen))) + if err != nil { + return 0, err + } + var flags int + if param.DataIO > 0 { + flags |= SCTP_EVENT_DATA_IO + } + if param.Association > 0 { + flags |= SCTP_EVENT_ASSOCIATION + } + if param.Address > 0 { + flags |= SCTP_EVENT_ADDRESS + } + if param.SendFailure > 0 { + flags |= SCTP_EVENT_SEND_FAILURE + } + if param.PeerError > 0 { + flags |= SCTP_EVENT_PEER_ERROR + } + if param.Shutdown > 0 { + flags |= SCTP_EVENT_SHUTDOWN + } + if param.PartialDelivery > 0 { + flags |= SCTP_EVENT_PARTIAL_DELIVERY + } + if param.AdaptationLayer > 0 { + flags |= SCTP_EVENT_ADAPTATION_LAYER + } + if param.Authentication > 0 { + flags |= SCTP_EVENT_AUTHENTICATION + } + if param.SenderDry > 0 { + flags |= SCTP_EVENT_SENDER_DRY + } + return flags, nil +} + +func (c *SCTPConn) SetDefaultSentParam(info *SndRcvInfo) error { + optlen := unsafe.Sizeof(*info) + _, _, err := setsockopt(c.fd(), SCTP_DEFAULT_SENT_PARAM, uintptr(unsafe.Pointer(info)), uintptr(optlen)) + return err +} + +func (c *SCTPConn) GetDefaultSentParam() (*SndRcvInfo, error) { + info := &SndRcvInfo{} + optlen := unsafe.Sizeof(*info) + _, _, err := getsockopt(c.fd(), SCTP_DEFAULT_SENT_PARAM, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(&optlen))) + return info, err +} + +func resolveFromRawAddr(ptr unsafe.Pointer, n int) (*SCTPAddr, error) { + addr := &SCTPAddr{ + IPAddrs: make([]net.IPAddr, n), + } + + switch family := (*(*syscall.RawSockaddrAny)(ptr)).Addr.Family; family { + case syscall.AF_INET: + addr.Port = int(ntohs(uint16((*(*syscall.RawSockaddrInet4)(ptr)).Port))) + tmp := syscall.RawSockaddrInet4{} + size := unsafe.Sizeof(tmp) + for i := 0; i < n; i++ { + a := *(*syscall.RawSockaddrInet4)(unsafe.Pointer( + uintptr(ptr) + size*uintptr(i))) + addr.IPAddrs[i] = net.IPAddr{IP: a.Addr[:]} + } + case syscall.AF_INET6: + addr.Port = int(ntohs(uint16((*(*syscall.RawSockaddrInet4)(ptr)).Port))) + tmp := syscall.RawSockaddrInet6{} + size := unsafe.Sizeof(tmp) + for i := 0; i < n; i++ { + a := *(*syscall.RawSockaddrInet6)(unsafe.Pointer( + uintptr(ptr) + size*uintptr(i))) + var zone string + ifi, err := net.InterfaceByIndex(int(a.Scope_id)) + if err == nil { + zone = ifi.Name + } + addr.IPAddrs[i] = net.IPAddr{IP: a.Addr[:], Zone: zone} + } + default: + return nil, fmt.Errorf("unknown address family: %d", family) + } + return addr, nil +} + +func sctpGetAddrs(fd, id, optname int) (*SCTPAddr, error) { + + type getaddrs struct { + assocId int32 + addrNum uint32 + addrs [4096]byte + } + param := getaddrs{ + assocId: int32(id), + } + optlen := unsafe.Sizeof(param) + _, _, err := getsockopt(fd, uintptr(optname), uintptr(unsafe.Pointer(¶m)), uintptr(unsafe.Pointer(&optlen))) + if err != nil { + return nil, err + } + return resolveFromRawAddr(unsafe.Pointer(¶m.addrs), int(param.addrNum)) +} + +func (c *SCTPConn) SCTPGetPrimaryPeerAddr() (*SCTPAddr, error) { + + type sctpGetSetPrim struct { + assocId int32 + addrs [128]byte + } + param := sctpGetSetPrim{ + assocId: int32(0), + } + optlen := unsafe.Sizeof(param) + _, _, err := getsockopt(c.fd(), SCTP_PRIMARY_ADDR, uintptr(unsafe.Pointer(¶m)), uintptr(unsafe.Pointer(&optlen))) + if err != nil { + return nil, err + } + return resolveFromRawAddr(unsafe.Pointer(¶m.addrs), 1) +} + +func (c *SCTPConn) SCTPLocalAddr(id int) (*SCTPAddr, error) { + return sctpGetAddrs(c.fd(), id, SCTP_GET_LOCAL_ADDRS) +} + +func (c *SCTPConn) SCTPRemoteAddr(id int) (*SCTPAddr, error) { + return sctpGetAddrs(c.fd(), id, SCTP_GET_PEER_ADDRS) +} + +func (c *SCTPConn) LocalAddr() net.Addr { + addr, err := sctpGetAddrs(c.fd(), 0, SCTP_GET_LOCAL_ADDRS) + if err != nil { + return nil + } + return addr +} + +func (c *SCTPConn) RemoteAddr() net.Addr { + addr, err := sctpGetAddrs(c.fd(), 0, SCTP_GET_PEER_ADDRS) + if err != nil { + return nil + } + return addr +} + +func (c *SCTPConn) PeelOff(id int) (*SCTPConn, error) { + type peeloffArg struct { + assocId int32 + sd int + } + param := peeloffArg{ + assocId: int32(id), + } + optlen := unsafe.Sizeof(param) + _, _, err := getsockopt(c.fd(), SCTP_SOCKOPT_PEELOFF, uintptr(unsafe.Pointer(¶m)), uintptr(unsafe.Pointer(&optlen))) + if err != nil { + return nil, err + } + return &SCTPConn{_fd: int32(param.sd)}, nil +} + +func (c *SCTPConn) SetDeadline(t time.Time) error { + return syscall.EOPNOTSUPP +} + +func (c *SCTPConn) SetReadDeadline(t time.Time) error { + return syscall.EOPNOTSUPP +} + +func (c *SCTPConn) SetWriteDeadline(t time.Time) error { + return syscall.EOPNOTSUPP +} + +type SCTPListener struct { + fd int + m sync.Mutex +} + +func (ln *SCTPListener) Addr() net.Addr { + laddr, err := sctpGetAddrs(ln.fd, 0, SCTP_GET_LOCAL_ADDRS) + if err != nil { + return nil + } + return laddr +} + +type SCTPSndRcvInfoWrappedConn struct { + conn *SCTPConn +} + +func NewSCTPSndRcvInfoWrappedConn(conn *SCTPConn) *SCTPSndRcvInfoWrappedConn { + conn.SubscribeEvents(SCTP_EVENT_DATA_IO) + return &SCTPSndRcvInfoWrappedConn{conn} +} + +func (c *SCTPSndRcvInfoWrappedConn) Write(b []byte) (int, error) { + if len(b) < int(sndRcvInfoSize) { + return 0, syscall.EINVAL + } + info := (*SndRcvInfo)(unsafe.Pointer(&b[0])) + n, err := c.conn.SCTPWrite(b[sndRcvInfoSize:], info) + return n + int(sndRcvInfoSize), err +} + +func (c *SCTPSndRcvInfoWrappedConn) Read(b []byte) (int, error) { + if len(b) < int(sndRcvInfoSize) { + return 0, syscall.EINVAL + } + n, info, err := c.conn.SCTPRead(b[sndRcvInfoSize:]) + if err != nil { + return n, err + } + copy(b, toBuf(info)) + return n + int(sndRcvInfoSize), err +} + +func (c *SCTPSndRcvInfoWrappedConn) Close() error { + return c.conn.Close() +} + +func (c *SCTPSndRcvInfoWrappedConn) LocalAddr() net.Addr { + return c.conn.LocalAddr() +} + +func (c *SCTPSndRcvInfoWrappedConn) RemoteAddr() net.Addr { + return c.conn.RemoteAddr() +} + +func (c *SCTPSndRcvInfoWrappedConn) SetDeadline(t time.Time) error { + return c.conn.SetDeadline(t) +} + +func (c *SCTPSndRcvInfoWrappedConn) SetReadDeadline(t time.Time) error { + return c.conn.SetReadDeadline(t) +} + +func (c *SCTPSndRcvInfoWrappedConn) SetWriteDeadline(t time.Time) error { + return c.conn.SetWriteDeadline(t) +} + +func (c *SCTPSndRcvInfoWrappedConn) SetWriteBuffer(bytes int) error { + return c.conn.SetWriteBuffer(bytes) +} + +func (c *SCTPSndRcvInfoWrappedConn) GetWriteBuffer() (int, error) { + return c.conn.GetWriteBuffer() +} + +func (c *SCTPSndRcvInfoWrappedConn) SetReadBuffer(bytes int) error { + return c.conn.SetReadBuffer(bytes) +} + +func (c *SCTPSndRcvInfoWrappedConn) GetReadBuffer() (int, error) { + return c.conn.GetReadBuffer() +} + +// SocketConfig contains options for the SCTP socket. +type SocketConfig struct { + // If Control is not nil it is called after the socket is created but before + // it is bound or connected. + Control func(network, address string, c syscall.RawConn) error + + // InitMsg is the options to send in the initial SCTP message + InitMsg InitMsg +} + +func (cfg *SocketConfig) Listen(net string, laddr *SCTPAddr) (*SCTPListener, error) { + return listenSCTPExtConfig(net, laddr, cfg.InitMsg, cfg.Control) +} + +func (cfg *SocketConfig) Dial(net string, laddr, raddr *SCTPAddr) (*SCTPConn, error) { + return dialSCTPExtConfig(net, laddr, raddr, cfg.InitMsg, cfg.Control) +} diff --git a/vendor/github.com/ishidawataru/sctp/sctp_linux.go b/vendor/github.com/ishidawataru/sctp/sctp_linux.go new file mode 100644 index 000000000000..ac340ddfbfa6 --- /dev/null +++ b/vendor/github.com/ishidawataru/sctp/sctp_linux.go @@ -0,0 +1,305 @@ +// +build linux,!386 +// Copyright 2019 Wataru Ishida. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sctp + +import ( + "io" + "net" + "sync/atomic" + "syscall" + "unsafe" +) + +func setsockopt(fd int, optname, optval, optlen uintptr) (uintptr, uintptr, error) { + // FIXME: syscall.SYS_SETSOCKOPT is undefined on 386 + r0, r1, errno := syscall.Syscall6(syscall.SYS_SETSOCKOPT, + uintptr(fd), + SOL_SCTP, + optname, + optval, + optlen, + 0) + if errno != 0 { + return r0, r1, errno + } + return r0, r1, nil +} + +func getsockopt(fd int, optname, optval, optlen uintptr) (uintptr, uintptr, error) { + // FIXME: syscall.SYS_GETSOCKOPT is undefined on 386 + r0, r1, errno := syscall.Syscall6(syscall.SYS_GETSOCKOPT, + uintptr(fd), + SOL_SCTP, + optname, + optval, + optlen, + 0) + if errno != 0 { + return r0, r1, errno + } + return r0, r1, nil +} + +type rawConn struct { + sockfd int +} + +func (r rawConn) Control(f func(fd uintptr)) error { + f(uintptr(r.sockfd)) + return nil +} + +func (r rawConn) Read(f func(fd uintptr) (done bool)) error { + panic("not implemented") +} + +func (r rawConn) Write(f func(fd uintptr) (done bool)) error { + panic("not implemented") +} + +func (c *SCTPConn) SCTPWrite(b []byte, info *SndRcvInfo) (int, error) { + var cbuf []byte + if info != nil { + cmsgBuf := toBuf(info) + hdr := &syscall.Cmsghdr{ + Level: syscall.IPPROTO_SCTP, + Type: SCTP_CMSG_SNDRCV, + } + + // bitwidth of hdr.Len is platform-specific, + // so we use hdr.SetLen() rather than directly setting hdr.Len + hdr.SetLen(syscall.CmsgSpace(len(cmsgBuf))) + cbuf = append(toBuf(hdr), cmsgBuf...) + } + return syscall.SendmsgN(c.fd(), b, cbuf, nil, 0) +} + +func parseSndRcvInfo(b []byte) (*SndRcvInfo, error) { + msgs, err := syscall.ParseSocketControlMessage(b) + if err != nil { + return nil, err + } + for _, m := range msgs { + if m.Header.Level == syscall.IPPROTO_SCTP { + switch m.Header.Type { + case SCTP_CMSG_SNDRCV: + return (*SndRcvInfo)(unsafe.Pointer(&m.Data[0])), nil + } + } + } + return nil, nil +} + +func (c *SCTPConn) SCTPRead(b []byte) (int, *SndRcvInfo, error) { + oob := make([]byte, 254) + for { + n, oobn, recvflags, _, err := syscall.Recvmsg(c.fd(), b, oob, 0) + if err != nil { + return n, nil, err + } + + if n == 0 && oobn == 0 { + return 0, nil, io.EOF + } + + if recvflags&MSG_NOTIFICATION > 0 && c.notificationHandler != nil { + if err := c.notificationHandler(b[:n]); err != nil { + return 0, nil, err + } + } else { + var info *SndRcvInfo + if oobn > 0 { + info, err = parseSndRcvInfo(oob[:oobn]) + } + return n, info, err + } + } +} + +func (c *SCTPConn) Close() error { + if c != nil { + fd := atomic.SwapInt32(&c._fd, -1) + if fd > 0 { + info := &SndRcvInfo{ + Flags: SCTP_EOF, + } + c.SCTPWrite(nil, info) + syscall.Shutdown(int(fd), syscall.SHUT_RDWR) + return syscall.Close(int(fd)) + } + } + return syscall.EBADF +} + +func (c *SCTPConn) SetWriteBuffer(bytes int) error { + return syscall.SetsockoptInt(c.fd(), syscall.SOL_SOCKET, syscall.SO_SNDBUF, bytes) +} + +func (c *SCTPConn) GetWriteBuffer() (int, error) { + return syscall.GetsockoptInt(c.fd(), syscall.SOL_SOCKET, syscall.SO_SNDBUF) +} + +func (c *SCTPConn) SetReadBuffer(bytes int) error { + return syscall.SetsockoptInt(c.fd(), syscall.SOL_SOCKET, syscall.SO_RCVBUF, bytes) +} + +func (c *SCTPConn) GetReadBuffer() (int, error) { + return syscall.GetsockoptInt(c.fd(), syscall.SOL_SOCKET, syscall.SO_RCVBUF) +} + +// ListenSCTP - start listener on specified address/port +func ListenSCTP(net string, laddr *SCTPAddr) (*SCTPListener, error) { + return ListenSCTPExt(net, laddr, InitMsg{NumOstreams: SCTP_MAX_STREAM}) +} + +// ListenSCTPExt - start listener on specified address/port with given SCTP options +func ListenSCTPExt(network string, laddr *SCTPAddr, options InitMsg) (*SCTPListener, error) { + return listenSCTPExtConfig(network, laddr, options, nil) +} + +// listenSCTPExtConfig - start listener on specified address/port with given SCTP options and socket configuration +func listenSCTPExtConfig(network string, laddr *SCTPAddr, options InitMsg, control func(network, address string, c syscall.RawConn) error) (*SCTPListener, error) { + af, ipv6only := favoriteAddrFamily(network, laddr, nil, "listen") + sock, err := syscall.Socket( + af, + syscall.SOCK_STREAM, + syscall.IPPROTO_SCTP, + ) + if err != nil { + return nil, err + } + + // close socket on error + defer func() { + if err != nil { + syscall.Close(sock) + } + }() + if err = setDefaultSockopts(sock, af, ipv6only); err != nil { + return nil, err + } + if control != nil { + rc := rawConn{sockfd: sock} + if err = control(network, laddr.String(), rc); err != nil { + return nil, err + } + } + err = setInitOpts(sock, options) + if err != nil { + return nil, err + } + + if laddr != nil { + // If IP address and/or port was not provided so far, let's use the unspecified IPv4 or IPv6 address + if len(laddr.IPAddrs) == 0 { + if af == syscall.AF_INET { + laddr.IPAddrs = append(laddr.IPAddrs, net.IPAddr{IP: net.IPv4zero}) + } else if af == syscall.AF_INET6 { + laddr.IPAddrs = append(laddr.IPAddrs, net.IPAddr{IP: net.IPv6zero}) + } + } + err := SCTPBind(sock, laddr, SCTP_BINDX_ADD_ADDR) + if err != nil { + return nil, err + } + } + err = syscall.Listen(sock, syscall.SOMAXCONN) + if err != nil { + return nil, err + } + return &SCTPListener{ + fd: sock, + }, nil +} + +// AcceptSCTP waits for and returns the next SCTP connection to the listener. +func (ln *SCTPListener) AcceptSCTP() (*SCTPConn, error) { + fd, _, err := syscall.Accept4(ln.fd, 0) + return NewSCTPConn(fd, nil), err +} + +// Accept waits for and returns the next connection connection to the listener. +func (ln *SCTPListener) Accept() (net.Conn, error) { + return ln.AcceptSCTP() +} + +func (ln *SCTPListener) Close() error { + syscall.Shutdown(ln.fd, syscall.SHUT_RDWR) + return syscall.Close(ln.fd) +} + +// DialSCTP - bind socket to laddr (if given) and connect to raddr +func DialSCTP(net string, laddr, raddr *SCTPAddr) (*SCTPConn, error) { + return DialSCTPExt(net, laddr, raddr, InitMsg{NumOstreams: SCTP_MAX_STREAM}) +} + +// DialSCTPExt - same as DialSCTP but with given SCTP options +func DialSCTPExt(network string, laddr, raddr *SCTPAddr, options InitMsg) (*SCTPConn, error) { + return dialSCTPExtConfig(network, laddr, raddr, options, nil) +} + +// dialSCTPExtConfig - same as DialSCTP but with given SCTP options and socket configuration +func dialSCTPExtConfig(network string, laddr, raddr *SCTPAddr, options InitMsg, control func(network, address string, c syscall.RawConn) error) (*SCTPConn, error) { + af, ipv6only := favoriteAddrFamily(network, laddr, raddr, "dial") + sock, err := syscall.Socket( + af, + syscall.SOCK_STREAM, + syscall.IPPROTO_SCTP, + ) + if err != nil { + return nil, err + } + + // close socket on error + defer func() { + if err != nil { + syscall.Close(sock) + } + }() + if err = setDefaultSockopts(sock, af, ipv6only); err != nil { + return nil, err + } + if control != nil { + rc := rawConn{sockfd: sock} + if err = control(network, laddr.String(), rc); err != nil { + return nil, err + } + } + err = setInitOpts(sock, options) + if err != nil { + return nil, err + } + if laddr != nil { + // If IP address and/or port was not provided so far, let's use the unspecified IPv4 or IPv6 address + if len(laddr.IPAddrs) == 0 { + if af == syscall.AF_INET { + laddr.IPAddrs = append(laddr.IPAddrs, net.IPAddr{IP: net.IPv4zero}) + } else if af == syscall.AF_INET6 { + laddr.IPAddrs = append(laddr.IPAddrs, net.IPAddr{IP: net.IPv6zero}) + } + } + err := SCTPBind(sock, laddr, SCTP_BINDX_ADD_ADDR) + if err != nil { + return nil, err + } + } + _, err = SCTPConnect(sock, raddr) + if err != nil { + return nil, err + } + return NewSCTPConn(sock, nil), nil +} diff --git a/vendor/github.com/ishidawataru/sctp/sctp_unsupported.go b/vendor/github.com/ishidawataru/sctp/sctp_unsupported.go new file mode 100644 index 000000000000..118fe159e92d --- /dev/null +++ b/vendor/github.com/ishidawataru/sctp/sctp_unsupported.go @@ -0,0 +1,98 @@ +// +build !linux linux,386 +// Copyright 2019 Wataru Ishida. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sctp + +import ( + "errors" + "net" + "runtime" + "syscall" +) + +var ErrUnsupported = errors.New("SCTP is unsupported on " + runtime.GOOS + "/" + runtime.GOARCH) + +func setsockopt(fd int, optname, optval, optlen uintptr) (uintptr, uintptr, error) { + return 0, 0, ErrUnsupported +} + +func getsockopt(fd int, optname, optval, optlen uintptr) (uintptr, uintptr, error) { + return 0, 0, ErrUnsupported +} + +func (c *SCTPConn) SCTPWrite(b []byte, info *SndRcvInfo) (int, error) { + return 0, ErrUnsupported +} + +func (c *SCTPConn) SCTPRead(b []byte) (int, *SndRcvInfo, error) { + return 0, nil, ErrUnsupported +} + +func (c *SCTPConn) Close() error { + return ErrUnsupported +} + +func (c *SCTPConn) SetWriteBuffer(bytes int) error { + return ErrUnsupported +} + +func (c *SCTPConn) GetWriteBuffer() (int, error) { + return 0, ErrUnsupported +} + +func (c *SCTPConn) SetReadBuffer(bytes int) error { + return ErrUnsupported +} + +func (c *SCTPConn) GetReadBuffer() (int, error) { + return 0, ErrUnsupported +} + +func ListenSCTP(net string, laddr *SCTPAddr) (*SCTPListener, error) { + return nil, ErrUnsupported +} + +func ListenSCTPExt(net string, laddr *SCTPAddr, options InitMsg) (*SCTPListener, error) { + return nil, ErrUnsupported +} + +func listenSCTPExtConfig(network string, laddr *SCTPAddr, options InitMsg, control func(network, address string, c syscall.RawConn) error) (*SCTPListener, error) { + return nil, ErrUnsupported +} + +func (ln *SCTPListener) Accept() (net.Conn, error) { + return nil, ErrUnsupported +} + +func (ln *SCTPListener) AcceptSCTP() (*SCTPConn, error) { + return nil, ErrUnsupported +} + +func (ln *SCTPListener) Close() error { + return ErrUnsupported +} + +func DialSCTP(net string, laddr, raddr *SCTPAddr) (*SCTPConn, error) { + return nil, ErrUnsupported +} + +func DialSCTPExt(network string, laddr, raddr *SCTPAddr, options InitMsg) (*SCTPConn, error) { + return nil, ErrUnsupported +} + +func dialSCTPExtConfig(network string, laddr, raddr *SCTPAddr, options InitMsg, control func(network, address string, c syscall.RawConn) error) (*SCTPConn, error) { + return nil, ErrUnsupported +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 07187680e576..e5c024a24073 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -254,6 +254,11 @@ github.com/docker/go-metrics # github.com/docker/go-units v0.4.0 ## explicit github.com/docker/go-units +# github.com/docker/libnetwork v0.8.0-dev.2.0.20200612180813-9e99af28df21 +## explicit +github.com/docker/libnetwork/resolvconf +github.com/docker/libnetwork/resolvconf/dns +github.com/docker/libnetwork/types # github.com/dustin/go-humanize v1.0.0 ## explicit github.com/dustin/go-humanize @@ -508,6 +513,9 @@ github.com/hpcloud/tail/ratelimiter github.com/hpcloud/tail/util github.com/hpcloud/tail/watch github.com/hpcloud/tail/winfile +# github.com/ishidawataru/sctp v0.0.0-20191218070446-00ab2ac2db07 +## explicit +github.com/ishidawataru/sctp # github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af github.com/jmespath/go-jmespath # github.com/joyent/triton-go v0.0.0-20190112182421-51ffac552869 diff --git a/website/pages/docs/job-specification/network.mdx b/website/pages/docs/job-specification/network.mdx index c6a9c1b17ccb..373f379da1c1 100644 --- a/website/pages/docs/job-specification/network.mdx +++ b/website/pages/docs/job-specification/network.mdx @@ -74,6 +74,10 @@ job "docs" { - `host` - Each task will join the host network namespace and a shared network namespace is not created. This matches the current behavior in Nomad 0.9. + - `dns` ([DNSConfig](#dns-parameters): nil) - Sets the DNS configuration + for the allocations. By default all DNS configuration is inherited from the client host. + DNS configuration is only supported on Linux clients at this time. + ### `port` Parameters - `static` `(int: nil)` - Specifies the static TCP/UDP port to allocate. If omitted, a dynamic port is chosen. We **do not recommend** using static ports, except @@ -98,6 +102,12 @@ When the task starts, it will be passed the following environment variables: The label of the port is just text - it has no special meaning to Nomad. +## `dns` Parameters + +- `servers` `(array: nil)` - Sets the DNS nameservers the allocation uses for name resolution. +- `searches` `(array: nil)` - Sets the search list for hostname lookup +- `options` `(array: nil)` - Sets internal resolver variables. + ## `network` Examples The following examples only show the `network` stanzas. Remember that the @@ -202,6 +212,18 @@ network { } ``` +### DNS + +The following example configures the allocation to use Google's DNS resolvers 8.8.8.8 and 8.8.4.4. + +```hcl +network { + dns { + servers = ["8.8.8.8", "8.8.4.4"] + } +} +``` + ### Limitations - Only one `network` stanza can be specified, when it is defined at the task group level.