Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

consul/connect: interpolate connect block #9586

Merged
merged 3 commits into from
Dec 9, 2020
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
## 1.0.1 (Unreleased)

IMPROVEMENTS:
* consul/connect: interpolate connect block [[GH-9586](https://github.com/hashicorp/nomad/pull/9586)]
shoenig marked this conversation as resolved.
Show resolved Hide resolved

## 1.0.0 (December 8, 2020)

FEATURES:
Expand Down
184 changes: 158 additions & 26 deletions client/taskenv/services.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ func InterpolateServices(taskEnv *TaskEnv, services []*structs.Service) []*struc
interpolated := make([]*structs.Service, len(services))

for i, origService := range services {
// Create a copy as we need to reinterpolate every time the
// environment changes
// Create a copy as we need to re-interpolate every time the
// environment changes.
service := origService.Copy()

for _, check := range service.Checks {
Expand All @@ -31,42 +31,174 @@ func InterpolateServices(taskEnv *TaskEnv, services []*structs.Service) []*struc
check.InitialStatus = taskEnv.ReplaceEnv(check.InitialStatus)
check.Method = taskEnv.ReplaceEnv(check.Method)
check.GRPCService = taskEnv.ReplaceEnv(check.GRPCService)
if len(check.Header) > 0 {
header := make(map[string][]string, len(check.Header))
for k, vs := range check.Header {
newVals := make([]string, len(vs))
for i, v := range vs {
newVals[i] = taskEnv.ReplaceEnv(v)
}
header[taskEnv.ReplaceEnv(k)] = newVals
}
check.Header = header
}
check.Header = interpolateMapStringSliceString(taskEnv, check.Header)
}

service.Name = taskEnv.ReplaceEnv(service.Name)
service.PortLabel = taskEnv.ReplaceEnv(service.PortLabel)
service.Tags = taskEnv.ParseAndReplace(service.Tags)
service.CanaryTags = taskEnv.ParseAndReplace(service.CanaryTags)
service.Meta = interpolateMapStringString(taskEnv, service.Meta)
service.CanaryMeta = interpolateMapStringString(taskEnv, service.CanaryMeta)
service.Connect = interpolateConnect(taskEnv, service.Connect)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Somewhat related because we're potentially setting these to nil and not empty maps, none of these three fields is Canonicalized. Should we be doing that and/or should we be assigning an empty map here for safety?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The canonicalized form of empty maps is nil (at least in services + consul)

// Ensure empty maps/slices are treated as null to avoid scheduling

Copy also make use of the helper.Copy* functions which also reduce to nil if there are no elements
https://github.com/hashicorp/nomad/blob/master/helper/funcs.go#L282


if len(service.Meta) > 0 {
meta := make(map[string]string, len(service.Meta))
for k, v := range service.Meta {
meta[k] = taskEnv.ReplaceEnv(v)
}
service.Meta = meta
interpolated[i] = service
}

return interpolated
}

func interpolateMapStringSliceString(taskEnv *TaskEnv, orig map[string][]string) map[string][]string {
if len(orig) == 0 {
return nil
}

m := make(map[string][]string, len(orig))
for k, vs := range orig {
m[taskEnv.ReplaceEnv(k)] = taskEnv.ParseAndReplace(vs)
}
return m
}

func interpolateMapStringString(taskEnv *TaskEnv, orig map[string]string) map[string]string {
if len(orig) == 0 {
return nil
}

m := make(map[string]string, len(orig))
for k, v := range orig {
m[taskEnv.ReplaceEnv(k)] = taskEnv.ReplaceEnv(v)
}
return m
}

func interpolateMapStringInterface(taskEnv *TaskEnv, orig map[string]interface{}) map[string]interface{} {
if len(orig) == 0 {
return nil
}

m := make(map[string]interface{}, len(orig))
for k, v := range orig {
m[taskEnv.ReplaceEnv(k)] = v
}
return m
}

func interpolateConnect(taskEnv *TaskEnv, orig *structs.ConsulConnect) *structs.ConsulConnect {
if orig == nil {
return nil
}

// make one copy and interpolate in-place on that
modified := orig.Copy()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The caller copies Service... shouldn't we already have a copy of the structs.ConsulConnect too?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch! Indeed there's no reason to create an extra copy here.

interpolateConnectSidecarService(taskEnv, modified.SidecarService)
interpolateConnectSidecarTask(taskEnv, modified.SidecarTask)
if modified.Gateway != nil {
interpolateConnectGatewayProxy(taskEnv, modified.Gateway.Proxy)
interpolateConnectGatewayIngress(taskEnv, modified.Gateway.Ingress)
}
return modified
}

func interpolateConnectGatewayProxy(taskEnv *TaskEnv, proxy *structs.ConsulGatewayProxy) {
if proxy == nil {
return
}

m := make(map[string]*structs.ConsulGatewayBindAddress, len(proxy.EnvoyGatewayBindAddresses))
for k, v := range proxy.EnvoyGatewayBindAddresses {
m[taskEnv.ReplaceEnv(k)] = &structs.ConsulGatewayBindAddress{
Address: taskEnv.ReplaceEnv(v.Address),
Port: v.Port,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like you interpolate the sidecar port below. Is there a reason we're not can't for this port?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The ConsulGatewayBindAddress field is an int, and there's (currently?) no way to interpolate int values
https://github.com/hashicorp/nomad/blob/master/nomad/structs/services.go#L1291

}
}

proxy.EnvoyGatewayBindAddresses = m
proxy.Config = interpolateMapStringInterface(taskEnv, proxy.Config)
}

func interpolateConnectGatewayIngress(taskEnv *TaskEnv, ingress *structs.ConsulIngressConfigEntry) {
if ingress == nil {
return
}

for _, listener := range ingress.Listeners {
listener.Protocol = taskEnv.ReplaceEnv(listener.Protocol)
for _, service := range listener.Services {
service.Name = taskEnv.ReplaceEnv(service.Name)
service.Hosts = taskEnv.ParseAndReplace(service.Hosts)
}
}
}

func interpolateConnectSidecarService(taskEnv *TaskEnv, sidecar *structs.ConsulSidecarService) {
if sidecar == nil {
return
}

if len(service.CanaryMeta) > 0 {
canaryMeta := make(map[string]string, len(service.CanaryMeta))
for k, v := range service.CanaryMeta {
canaryMeta[k] = taskEnv.ReplaceEnv(v)
sidecar.Port = taskEnv.ReplaceEnv(sidecar.Port)
sidecar.Tags = taskEnv.ParseAndReplace(sidecar.Tags)
if sidecar.Proxy != nil {
sidecar.Proxy.LocalServiceAddress = taskEnv.ReplaceEnv(sidecar.Proxy.LocalServiceAddress)
if sidecar.Proxy.Expose != nil {
for i := 0; i < len(sidecar.Proxy.Expose.Paths); i++ {
sidecar.Proxy.Expose.Paths[i].Protocol = taskEnv.ReplaceEnv(sidecar.Proxy.Expose.Paths[i].Protocol)
sidecar.Proxy.Expose.Paths[i].ListenerPort = taskEnv.ReplaceEnv(sidecar.Proxy.Expose.Paths[i].ListenerPort)
sidecar.Proxy.Expose.Paths[i].Path = taskEnv.ReplaceEnv(sidecar.Proxy.Expose.Paths[i].Path)
}
service.CanaryMeta = canaryMeta
}
for i := 0; i < len(sidecar.Proxy.Upstreams); i++ {
sidecar.Proxy.Upstreams[i].Datacenter = taskEnv.ReplaceEnv(sidecar.Proxy.Upstreams[i].Datacenter)
sidecar.Proxy.Upstreams[i].DestinationName = taskEnv.ReplaceEnv(sidecar.Proxy.Upstreams[i].DestinationName)
}
sidecar.Proxy.Config = interpolateMapStringInterface(taskEnv, sidecar.Proxy.Config)
}
}

interpolated[i] = service
func interpolateConnectSidecarTask(taskEnv *TaskEnv, task *structs.SidecarTask) {
if task == nil {
return
}

return interpolated
task.Driver = taskEnv.ReplaceEnv(task.Driver)
task.Config = interpolateMapStringInterface(taskEnv, task.Config)
task.Env = interpolateMapStringString(taskEnv, task.Env)
task.KillSignal = taskEnv.ReplaceEnv(task.KillSignal)
task.Meta = interpolateMapStringString(taskEnv, task.Meta)
interpolateTaskResources(taskEnv, task.Resources)
task.User = taskEnv.ReplaceEnv(task.User)
}

func interpolateTaskResources(taskEnv *TaskEnv, resources *structs.Resources) {
if resources == nil {
return
}

for i := 0; i < len(resources.Devices); i++ {
resources.Devices[i].Name = taskEnv.ReplaceEnv(resources.Devices[i].Name)
// do not interpolate constraints & affinities
}

for i := 0; i < len(resources.Networks); i++ {
resources.Networks[i].CIDR = taskEnv.ReplaceEnv(resources.Networks[i].CIDR)
resources.Networks[i].Device = taskEnv.ReplaceEnv(resources.Networks[i].Device)
resources.Networks[i].IP = taskEnv.ReplaceEnv(resources.Networks[i].IP)
resources.Networks[i].Mode = taskEnv.ReplaceEnv(resources.Networks[i].Mode)

if resources.Networks[i].DNS != nil {
resources.Networks[i].DNS.Options = taskEnv.ParseAndReplace(resources.Networks[i].DNS.Options)
resources.Networks[i].DNS.Searches = taskEnv.ParseAndReplace(resources.Networks[i].DNS.Searches)
resources.Networks[i].DNS.Servers = taskEnv.ParseAndReplace(resources.Networks[i].DNS.Servers)
}

for p := 0; p < len(resources.Networks[i].DynamicPorts); p++ {
resources.Networks[i].DynamicPorts[p].HostNetwork = taskEnv.ReplaceEnv(resources.Networks[i].DynamicPorts[p].HostNetwork)
resources.Networks[i].DynamicPorts[p].Label = taskEnv.ReplaceEnv(resources.Networks[i].DynamicPorts[p].Label)
}

for p := 0; p < len(resources.Networks[i].ReservedPorts); p++ {
resources.Networks[i].ReservedPorts[p].HostNetwork = taskEnv.ReplaceEnv(resources.Networks[i].ReservedPorts[p].HostNetwork)
resources.Networks[i].ReservedPorts[p].Label = taskEnv.ReplaceEnv(resources.Networks[i].ReservedPorts[p].Label)
}
}
}
Loading