Skip to content

Commit

Permalink
Merge pull request #7323 from hashicorp/f-connect-expose-checks
Browse files Browse the repository at this point in the history
 connect: enable proxy.expose configuration
  • Loading branch information
shoenig committed Mar 31, 2020
2 parents fe732fe + ee3b43e commit b6fe1e2
Show file tree
Hide file tree
Showing 16 changed files with 1,198 additions and 207 deletions.
17 changes: 15 additions & 2 deletions api/services.go
Original file line number Diff line number Diff line change
Expand Up @@ -168,8 +168,9 @@ type SidecarTask struct {

// ConsulProxy represents a Consul Connect sidecar proxy jobspec stanza.
type ConsulProxy struct {
LocalServiceAddress string `mapstructure:"local_service_address"`
LocalServicePort int `mapstructure:"local_service_port"`
LocalServiceAddress string `mapstructure:"local_service_address"`
LocalServicePort int `mapstructure:"local_service_port"`
ExposeConfig *ConsulExposeConfig `mapstructure:"expose"`
Upstreams []*ConsulUpstream
Config map[string]interface{}
}
Expand All @@ -179,3 +180,15 @@ type ConsulUpstream struct {
DestinationName string `mapstructure:"destination_name"`
LocalBindPort int `mapstructure:"local_bind_port"`
}

type ConsulExposeConfig struct {
Path []*ConsulExposePath `mapstructure:"path"`
// todo(shoenig): add magic for 'checks' option
}

type ConsulExposePath struct {
Path string
Protocol string
LocalPathPort int `mapstructure:"local_path_port"`
ListenerPort string `mapstructure:"listener_port"`
}
6 changes: 2 additions & 4 deletions client/allocrunner/networking_bridge_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,6 @@ func newBridgeNetworkConfigurator(log hclog.Logger, bridgeName, ipRange, cniPath

// ensureForwardingRules ensures that a forwarding rule is added to iptables
// to allow traffic inbound to the bridge network
// // ensureForwardingRules ensures that a forwarding rule is added to iptables
// to allow traffic inbound to the bridge network
func (b *bridgeNetworkConfigurator) ensureForwardingRules() error {
ipt, err := iptables.New()
if err != nil {
Expand Down Expand Up @@ -154,9 +152,9 @@ func (b *bridgeNetworkConfigurator) Setup(ctx context.Context, alloc *structs.Al
return err
}

// Depending on the version of bridge cni plugin used, a known race could occure
// Depending on the version of bridge cni plugin (< 0.8.4) a known race could occur
// where two alloc attempt to create the nomad bridge at the same time, resulting
// in one of them to fail. This rety attempts to overcome any
// in one of them to fail. This retry attempts to overcome those erroneous failures.
const retry = 3
for attempt := 1; ; attempt++ {
//TODO eventually returning the IP from the result would be nice to have in the alloc
Expand Down
2 changes: 1 addition & 1 deletion command/agent/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -587,7 +587,7 @@ func convertClientConfig(agentConfig *Config) (*clientconfig.Config, error) {
conf.ACLTokenTTL = agentConfig.ACL.TokenTTL
conf.ACLPolicyTTL = agentConfig.ACL.PolicyTTL

// Setup networking configration
// Setup networking configuration
conf.CNIPath = agentConfig.Client.CNIPath
conf.BridgeNetworkName = agentConfig.Client.BridgeNetworkName
conf.BridgeNetworkAllocSubnet = agentConfig.Client.BridgeNetworkSubnet
Expand Down
93 changes: 3 additions & 90 deletions command/agent/consul/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -1280,7 +1280,7 @@ func MakeCheckID(serviceID string, check *structs.ServiceCheck) string {
// createCheckReg creates a Check that can be registered with Consul.
//
// Script checks simply have a TTL set and the caller is responsible for
// running the script and heartbeating.
// running the script and heart-beating.
func createCheckReg(serviceID, checkID string, check *structs.ServiceCheck, host string, port int) (*api.AgentCheckRegistration, error) {
chkReg := api.AgentCheckRegistration{
ID: checkID,
Expand Down Expand Up @@ -1313,8 +1313,8 @@ func createCheckReg(serviceID, checkID string, check *structs.ServiceCheck, host
if err != nil {
return nil, err
}
url := base.ResolveReference(relative)
chkReg.HTTP = url.String()
checkURL := base.ResolveReference(relative)
chkReg.HTTP = checkURL.String()
chkReg.Method = check.Method
chkReg.Header = check.Header

Expand Down Expand Up @@ -1471,90 +1471,3 @@ func getAddress(addrMode, portLabel string, networks structs.Networks, driverNet
return "", 0, fmt.Errorf("invalid address mode %q", addrMode)
}
}

// newConnect creates a new Consul AgentServiceConnect struct based on a Nomad
// Connect struct. If the nomad Connect struct is nil, nil will be returned to
// disable Connect for this service.
func newConnect(serviceName string, nc *structs.ConsulConnect, networks structs.Networks) (*api.AgentServiceConnect, error) {
if nc == nil {
// No Connect stanza, returning nil is fine
return nil, nil
}

cc := &api.AgentServiceConnect{
Native: nc.Native,
}

if nc.SidecarService == nil {
return cc, nil
}

net, port, err := getConnectPort(serviceName, networks)
if err != nil {
return nil, err
}

// Bind to netns IP(s):port
proxyConfig := map[string]interface{}{}
localServiceAddress := ""
localServicePort := 0
if nc.SidecarService.Proxy != nil {
localServiceAddress = nc.SidecarService.Proxy.LocalServiceAddress
localServicePort = nc.SidecarService.Proxy.LocalServicePort
if nc.SidecarService.Proxy.Config != nil {
proxyConfig = nc.SidecarService.Proxy.Config
}
}
proxyConfig["bind_address"] = "0.0.0.0"
proxyConfig["bind_port"] = port.To

// Advertise host IP:port
cc.SidecarService = &api.AgentServiceRegistration{
Tags: helper.CopySliceString(nc.SidecarService.Tags),
Address: net.IP,
Port: port.Value,

// Automatically configure the proxy to bind to all addresses
// within the netns.
Proxy: &api.AgentServiceConnectProxyConfig{
LocalServiceAddress: localServiceAddress,
LocalServicePort: localServicePort,
Config: proxyConfig,
},
}

// If no further proxy settings were explicitly configured, exit early
if nc.SidecarService.Proxy == nil {
return cc, nil
}

numUpstreams := len(nc.SidecarService.Proxy.Upstreams)
if numUpstreams == 0 {
return cc, nil
}

upstreams := make([]api.Upstream, numUpstreams)
for i, nu := range nc.SidecarService.Proxy.Upstreams {
upstreams[i].DestinationName = nu.DestinationName
upstreams[i].LocalBindPort = nu.LocalBindPort
}
cc.SidecarService.Proxy.Upstreams = upstreams

return cc, nil
}

// getConnectPort returns the network and port for the Connect proxy sidecar
// defined for this service. An error is returned if the network and port
// cannot be determined.
func getConnectPort(serviceName string, networks structs.Networks) (*structs.NetworkResource, structs.Port, error) {
if n := len(networks); n != 1 {
return nil, structs.Port{}, fmt.Errorf("Connect only supported with exactly 1 network (found %d)", n)
}

port, ok := networks[0].PortForService(serviceName)
if !ok {
return nil, structs.Port{}, fmt.Errorf("No Connect port defined for service %q", serviceName)
}

return networks[0], port, nil
}
176 changes: 176 additions & 0 deletions command/agent/consul/connect.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
package consul

import (
"fmt"

"github.com/hashicorp/consul/api"
"github.com/hashicorp/nomad/helper"
"github.com/hashicorp/nomad/nomad/structs"
)

// newConnect creates a new Consul AgentServiceConnect struct based on a Nomad
// Connect struct. If the nomad Connect struct is nil, nil will be returned to
// disable Connect for this service.
func newConnect(serviceName string, nc *structs.ConsulConnect, networks structs.Networks) (*api.AgentServiceConnect, error) {
if nc == nil {
// no connect stanza means there is no connect service to register
return nil, nil
}

if nc.Native {
return &api.AgentServiceConnect{Native: true}, nil
}

sidecarReg, err := connectSidecarRegistration(serviceName, nc.SidecarService, networks)
if err != nil {
return nil, err
}

return &api.AgentServiceConnect{
Native: false,
SidecarService: sidecarReg,
}, nil
}

func connectSidecarRegistration(serviceName string, css *structs.ConsulSidecarService, networks structs.Networks) (*api.AgentServiceRegistration, error) {
if css == nil {
// no sidecar stanza means there is no sidecar service to register
return nil, nil
}

cNet, cPort, err := connectPort(serviceName, networks)
if err != nil {
return nil, err
}

proxy, err := connectProxy(css.Proxy, cPort.To, networks)
if err != nil {
return nil, err
}

return &api.AgentServiceRegistration{
Tags: helper.CopySliceString(css.Tags),
Port: cPort.Value,
Address: cNet.IP,
Proxy: proxy,
}, nil
}

func connectProxy(proxy *structs.ConsulProxy, cPort int, networks structs.Networks) (*api.AgentServiceConnectProxyConfig, error) {
if proxy == nil {
return nil, nil
}

expose, err := connectProxyExpose(proxy.Expose, networks)
if err != nil {
return nil, err
}

return &api.AgentServiceConnectProxyConfig{
LocalServiceAddress: proxy.LocalServiceAddress,
LocalServicePort: proxy.LocalServicePort,
Config: connectProxyConfig(proxy.Config, cPort),
Upstreams: connectUpstreams(proxy.Upstreams),
Expose: expose,
}, nil
}

func connectProxyExpose(expose *structs.ConsulExposeConfig, networks structs.Networks) (api.ExposeConfig, error) {
if expose == nil {
return api.ExposeConfig{}, nil
}

paths, err := connectProxyExposePaths(expose.Paths, networks)
if err != nil {
return api.ExposeConfig{}, err
}

return api.ExposeConfig{
Checks: false,
Paths: paths,
}, nil
}

func connectProxyExposePaths(in []structs.ConsulExposePath, networks structs.Networks) ([]api.ExposePath, error) {
if len(in) == 0 {
return nil, nil
}

paths := make([]api.ExposePath, len(in))
for i, path := range in {
if _, exposedPort, err := connectExposePathPort(path.ListenerPort, networks); err != nil {
return nil, err
} else {
paths[i] = api.ExposePath{
ListenerPort: exposedPort,
Path: path.Path,
LocalPathPort: path.LocalPathPort,
Protocol: path.Protocol,
ParsedFromCheck: false,
}
}
}
return paths, nil
}

func connectUpstreams(in []structs.ConsulUpstream) []api.Upstream {
if len(in) == 0 {
return nil
}

upstreams := make([]api.Upstream, len(in))
for i, upstream := range in {
upstreams[i] = api.Upstream{
DestinationName: upstream.DestinationName,
LocalBindPort: upstream.LocalBindPort,
}
}
return upstreams
}

func connectProxyConfig(cfg map[string]interface{}, port int) map[string]interface{} {
if cfg == nil {
cfg = make(map[string]interface{})
}
cfg["bind_address"] = "0.0.0.0"
cfg["bind_port"] = port
return cfg
}

func connectNetworkInvariants(networks structs.Networks) error {
if n := len(networks); n != 1 {
return fmt.Errorf("Connect only supported with exactly 1 network (found %d)", n)
}
return nil
}

// connectPort returns the network and port for the Connect proxy sidecar
// defined for this service. An error is returned if the network and port
// cannot be determined.
func connectPort(serviceName string, networks structs.Networks) (*structs.NetworkResource, structs.Port, error) {
if err := connectNetworkInvariants(networks); err != nil {
return nil, structs.Port{}, err
}

port, ok := networks[0].PortForService(serviceName)
if !ok {
return nil, structs.Port{}, fmt.Errorf("No Connect port defined for service %q", serviceName)
}

return networks[0], port, nil
}

// connectExposePathPort returns the port for the exposed path for the exposed
// proxy path.
func connectExposePathPort(portLabel string, networks structs.Networks) (string, int, error) {
if err := connectNetworkInvariants(networks); err != nil {
return "", 0, err
}

ip, port := networks.Port(portLabel)
if port == 0 {
return "", 0, fmt.Errorf("No port of label %q defined", portLabel)
}

return ip, port, nil
}
Loading

0 comments on commit b6fe1e2

Please sign in to comment.